DEV-374: implemented logical restore-test to check if restore was succussful

4 years ago · 6fbc3af3c4
parent 197bcfd4ea
commit 6fbc3af3c4
11 changed files with 224 additions and 11 deletions
--- a/restore-remote-database-backup.yml
+++ b/restore-remote-database-backup.yml
@ -168,6 +168,50 @@
      become: yes
      shell: '/root/restore.sh {{ stage }}'

+    - name: "Check for test data on postgres"
+      block:
+
+      - name: "Querying postgres ..."
+        become: yes
+        become_user: postgres
+        community.postgresql.postgresql_query:
+          db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
+          named_args:
+            quote_val: 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'postgres'
+
+    - name: "Check for test data on mariadb"
+      block:
+
+      - name: "Querying mariadb ..."
+        become: yes
+        become_user: root
+        community.mysql.mysql_query:
+          login_user: '{{ mysql_root_username }}'
+          login_password: "{{ mysql_root_password }}"
+          config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+          login_db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %s
+          positional_args:
+            - 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'maria'
+
+
 # TODO delete restore-server

 #############################################################
--- a/roles/backup/files/pull_remote_backups.sh
+++ b/roles/backup/files/pull_remote_backups.sh
@ -11,3 +11,6 @@ DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/

 mkdir -p ${DEST_DIR}
 rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/
+
+# remove files oder than XX in backup-DIR
+find ${DEST_DIR} -ctime +7 -delete
--- a/roles/hcloud/tasks/_set_server_state.yml
+++ b/roles/hcloud/tasks/_set_server_state.yml
@ -29,6 +29,7 @@
  poll: 15
  when:
    - server_state == 'present'
+    - not hcloud_response.msg is defined

 # just catching hetzner error and trying to avoid manual interaction
 - name: "DEBUGGING Block for hetzner << message not found >> - error"
--- a/roles/maria/files/testdb.sql
+++ b/roles/maria/files/testdb.sql
@ -0,0 +1,52 @@
+-- MySQL dump 10.19  Distrib 10.3.34-MariaDB, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: stararchitekt
+-- ------------------------------------------------------
+-- Server version       10.3.34-MariaDB-0ubuntu0.20.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `movie_quotes`
+--
+
+DROP TABLE IF EXISTS `movie_quotes`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `movie_quotes` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `movie` varchar(255) DEFAULT NULL,
+  `quote` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `movie_quotes`
+--
+
+LOCK TABLES `movie_quotes` WRITE;
+/*!40000 ALTER TABLE `movie_quotes` DISABLE KEYS */;
+INSERT INTO `movie_quotes` VALUES (1,'wargames','Shall we play');
+/*!40000 ALTER TABLE `movie_quotes` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-03-31  9:46:16
--- a/roles/maria/handlers/main.yml
+++ b/roles/maria/handlers/main.yml
@ -0,0 +1,6 @@
+---
+- name: "prometheus-mysqld-exporter restart"
+  service:
+    name: prometheus-mysqld-exporter
+    state: restarted
+    enabled: yes
--- a/roles/maria/tasks/main.yml
+++ b/roles/maria/tasks/main.yml
@ -14,12 +14,6 @@
    - python3-pymysql
    - prometheus-mysqld-exporter

- name: Stop prometheus-mysqld-exporter
-  service:
-    name: prometheus-mysqld-exporter
-    state: stopped
-    enabled: no
-
 - name: Fix binding..
  ansible.builtin.lineinfile:
    path: /etc/mysql/mariadb.conf.d/50-server.cnf
@ -77,6 +71,8 @@
    priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT"
    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
    login_password: "{{ mysql_root_password }}"
+  register: mysql_exporter_user_creds
+  notify: prometheus-mysqld-exporter restart

 - name: Ensure is prometheus-mysqld-exporter configured
  lineinfile:
@ -84,6 +80,7 @@
    line: 'DATA_SOURCE_NAME="prometheus@unix(/run/mysqld/mysqld.sock)/"'
    path: /etc/default/prometheus-mysqld-exporter
  register: mysql_exporter_data_source
+  notify: prometheus-mysqld-exporter restart

 - name: Setup prometheus-mysqld-exporter interface bind
  lineinfile:
@ -91,8 +88,9 @@
    regex: "^ARGS="
    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_maria }}'\""
  register: mysql_exporter_args
+  notify: prometheus-mysqld-exporter restart

- name: Start prometheus-mysqld-exporter if needed
+- name: "Ensure prometheus-mysqld-exporter is running"
  service:
    name: prometheus-mysqld-exporter
    state: started
@ -104,6 +102,25 @@
    path: '{{ backup_directory }}'
    mode: 0755

+- name: "Copy testdb.sql to ensure test DB"
+  copy:
+    src: '{{ item }}'
+    dest: '/tmp/{{ item }}'
+    mode: '0444'
+    owner: root
+    group: root
+  loop:
+    - testdb.sql
+
+- name: "Ensure test DB"
+  community.mysql.mysql_db:
+    login_user: '{{ mysql_root_username }}'
+    login_password: "{{ mysql_root_password }}"
+    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    name: dummytestdb
+    state: import
+    target: /tmp/testdb.sql
+
 - name: "Block: gpg stuff"
  block:
    - name: Create temp dir
--- a/roles/postgres/files/testdb.sql
+++ b/roles/postgres/files/testdb.sql
@ -0,0 +1,60 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 13.6 (Ubuntu 13.6-1.pgdg20.04+1)
+-- Dumped by pg_dump version 13.6 (Ubuntu 13.6-1.pgdg20.04+1)
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
+
+
+--
+-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: 
+--
+
+COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions';
+
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: movie_quotes; Type: TABLE; Schema: public; Owner: postgres
+--
+
+CREATE TABLE public.movie_quotes (
+    movie character varying(255),
+    quote character varying(255)
+);
+
+
+ALTER TABLE public.movie_quotes OWNER TO postgres;
+
+--
+-- Data for Name: movie_quotes; Type: TABLE DATA; Schema: public; Owner: postgres
+--
+
+COPY public.movie_quotes (movie, quote) FROM stdin;
+wargames	Shall we play
+\.
+
+
+--
+-- PostgreSQL database dump complete
+--
--- a/roles/postgres/tasks/base-requirements.yml
+++ b/roles/postgres/tasks/base-requirements.yml
@ -30,12 +30,15 @@
    repo: deb http://apt.postgresql.org/pub/repos/apt {{ default_postgres_target_distribution }} main
    state: present

- name: "Ensuring postgres is installed"
+- name: "Ensuring several packages being installed"
  apt:
-    name: "postgresql-{{ default_postgres_version }}"
+    name: "{{ item }}"
    update_cache: yes
    cache_valid_time: 900
    state: present
+  loop:
+    - 'postgresql-{{ default_postgres_version }}'
+    - python3-psycopg2

 - name: "Setup listen interfaces for postgresql instance"
  lineinfile:
--- a/roles/postgres/tasks/master-requirements.yml
+++ b/roles/postgres/tasks/master-requirements.yml
@ -157,3 +157,27 @@
  shell: "/usr/bin/psql -Atc \"SELECT pg_create_physical_replication_slot('pgstandby1');\""
  ignore_errors: yes # noqa ignore-errors
  when: replication_slot_check.stdout == "0"
+
+- name: "Copy testdb.sql to ensure test DB"
+  copy:
+    src: '{{ item }}'
+    dest: '/tmp/{{ item }}'
+    mode: '0444'
+    owner: postgres
+    group: postgres
+  loop:
+    - testdb.sql
+
+- name: "Ensure test DB"
+  become: yes
+  become_user: postgres
+  community.postgresql.postgresql_db:
+    name: dummytestdb
+
+- name: "Ensure content for test DB"
+  become: yes
+  become_user: postgres
+  community.postgresql.postgresql_db:
+    name: dummytestdb
+    state: restore
+    target: /tmp/testdb.sql
--- a/roles/restore_maria/files/restore.sh
+++ b/roles/restore_maria/files/restore.sh
@ -12,6 +12,7 @@ DATE=$(date +%F)

 systemctl stop mariadb

+rm -rf ${DATADIR}_moved
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}

--- a/roles/restore_postgres/files/restore.sh
+++ b/roles/restore_postgres/files/restore.sh
@ -13,6 +13,8 @@ PG_GROUP=postgres

 systemctl stop postgresql

+rm -rf ${DATADIR}_moved
+
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}