DEV-374: implemented logical restore-test to check if restore was succussful

restore-remote-database-backup.yml

@@ -168,6 +168,50 @@
       become: yes
       shell: '/root/restore.sh {{ stage }}'
 
+    - name: "Check for test data on postgres"
+      block:
+
+      - name: "Querying postgres ..."
+        become: yes
+        become_user: postgres
+        community.postgresql.postgresql_query:
+          db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
+          named_args:
+            quote_val: 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'postgres'
+
+    - name: "Check for test data on mariadb"
+      block:
+
+      - name: "Querying mariadb ..."
+        become: yes
+        become_user: root
+        community.mysql.mysql_query:
+          login_user: '{{ mysql_root_username }}'
+          login_password: "{{ mysql_root_password }}"
+          config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+          login_db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %s
+          positional_args:
+            - 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'maria'
+
+
 # TODO delete restore-server
 
 #############################################################

roles/backup/files/pull_remote_backups.sh

@@ -11,3 +11,6 @@ DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/
 
 mkdir -p ${DEST_DIR}
 rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/
+
+# remove files oder than XX in backup-DIR
+find ${DEST_DIR} -ctime +7 -delete

roles/hcloud/tasks/_set_server_state.yml

@@ -29,6 +29,7 @@
   poll: 15
   when:
     - server_state == 'present'
+    - not hcloud_response.msg is defined
 
 # just catching hetzner error and trying to avoid manual interaction
 - name: "DEBUGGING Block for hetzner << message not found >> - error"

roles/maria/files/testdb.sql

@@ -0,0 +1,52 @@
+-- MySQL dump 10.19  Distrib 10.3.34-MariaDB, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: stararchitekt
+-- ------------------------------------------------------
+-- Server version       10.3.34-MariaDB-0ubuntu0.20.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `movie_quotes`
+--
+
+DROP TABLE IF EXISTS `movie_quotes`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `movie_quotes` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `movie` varchar(255) DEFAULT NULL,
+  `quote` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `movie_quotes`
+--
+
+LOCK TABLES `movie_quotes` WRITE;
+/*!40000 ALTER TABLE `movie_quotes` DISABLE KEYS */;
+INSERT INTO `movie_quotes` VALUES (1,'wargames','Shall we play');
+/*!40000 ALTER TABLE `movie_quotes` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-03-31  9:46:16

roles/maria/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: "prometheus-mysqld-exporter restart"
+  service:
+    name: prometheus-mysqld-exporter
+    state: restarted
+    enabled: yes

roles/maria/tasks/main.yml

@@ -14,12 +14,6 @@
     - python3-pymysql
     - prometheus-mysqld-exporter
 
-- name: Stop prometheus-mysqld-exporter
-  service:
-    name: prometheus-mysqld-exporter
-    state: stopped
-    enabled: no
-
 - name: Fix binding..
   ansible.builtin.lineinfile:
     path: /etc/mysql/mariadb.conf.d/50-server.cnf
@@ -77,6 +71,8 @@
     priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT"
     config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
     login_password: "{{ mysql_root_password }}"
+  register: mysql_exporter_user_creds
+  notify: prometheus-mysqld-exporter restart
 
 - name: Ensure is prometheus-mysqld-exporter configured
   lineinfile:
@@ -84,6 +80,7 @@
     line: 'DATA_SOURCE_NAME="prometheus@unix(/run/mysqld/mysqld.sock)/"'
     path: /etc/default/prometheus-mysqld-exporter
   register: mysql_exporter_data_source
+  notify: prometheus-mysqld-exporter restart
 
 - name: Setup prometheus-mysqld-exporter interface bind
   lineinfile:
@@ -91,8 +88,9 @@
     regex: "^ARGS="
     line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_maria }}'\""
   register: mysql_exporter_args
+  notify: prometheus-mysqld-exporter restart
 
-- name: Start prometheus-mysqld-exporter if needed
+- name: "Ensure prometheus-mysqld-exporter is running"
   service:
     name: prometheus-mysqld-exporter
     state: started
@@ -104,6 +102,25 @@
     path: '{{ backup_directory }}'
     mode: 0755
 
+- name: "Copy testdb.sql to ensure test DB"
+  copy:
+    src: '{{ item }}'
+    dest: '/tmp/{{ item }}'
+    mode: '0444'
+    owner: root
+    group: root
+  loop:
+    - testdb.sql
+
+- name: "Ensure test DB"
+  community.mysql.mysql_db:
+    login_user: '{{ mysql_root_username }}'
+    login_password: "{{ mysql_root_password }}"
+    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    name: dummytestdb
+    state: import
+    target: /tmp/testdb.sql
+
 - name: "Block: gpg stuff"
   block:
     - name: Create temp dir
@@ -112,13 +129,13 @@
         suffix: gitcheckout
         path: /tmp
       register: tempdir
-    
+
     - name: "Checkout repo for gpg communication-keys"
       ansible.builtin.git:
         repo: 'https://{{ gituser | default("gitea-admin") | urlencode }}:{{ gitea_admin_password | urlencode }}@{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys.git'
         dest: '{{ tempdir.path }}'
         version: master
-    
+
 # there is no ansible gpg module already in place
 # linting violation needs to be whitelisted
     - name: "Importing stage specific automation gpg-key" # noqa command-instead-of-shell

roles/postgres/files/testdb.sql

@@ -0,0 +1,60 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 13.6 (Ubuntu 13.6-1.pgdg20.04+1)
+-- Dumped by pg_dump version 13.6 (Ubuntu 13.6-1.pgdg20.04+1)
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = on;
+SELECT pg_catalog.set_config('search_path', '', false);
+SET check_function_bodies = false;
+SET xmloption = content;
+SET client_min_messages = warning;
+SET row_security = off;
+
+--
+-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
+
+
+--
+-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: 
+--
+
+COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions';
+
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+--
+-- Name: movie_quotes; Type: TABLE; Schema: public; Owner: postgres
+--
+
+CREATE TABLE public.movie_quotes (
+    movie character varying(255),
+    quote character varying(255)
+);
+
+
+ALTER TABLE public.movie_quotes OWNER TO postgres;
+
+--
+-- Data for Name: movie_quotes; Type: TABLE DATA; Schema: public; Owner: postgres
+--
+
+COPY public.movie_quotes (movie, quote) FROM stdin;
+wargames	Shall we play
+\.
+
+
+--
+-- PostgreSQL database dump complete
+--

roles/postgres/tasks/base-requirements.yml

@@ -30,12 +30,15 @@
     repo: deb http://apt.postgresql.org/pub/repos/apt {{ default_postgres_target_distribution }} main
     state: present
 
-- name: "Ensuring postgres is installed"
+- name: "Ensuring several packages being installed"
   apt:
-    name: "postgresql-{{ default_postgres_version }}"
+    name: "{{ item }}"
     update_cache: yes
     cache_valid_time: 900
     state: present
+  loop:
+    - 'postgresql-{{ default_postgres_version }}'
+    - python3-psycopg2
 
 - name: "Setup listen interfaces for postgresql instance"
   lineinfile:

roles/postgres/tasks/master-requirements.yml

@@ -157,3 +157,27 @@
   shell: "/usr/bin/psql -Atc \"SELECT pg_create_physical_replication_slot('pgstandby1');\""
   ignore_errors: yes # noqa ignore-errors
   when: replication_slot_check.stdout == "0"
+
+- name: "Copy testdb.sql to ensure test DB"
+  copy:
+    src: '{{ item }}'
+    dest: '/tmp/{{ item }}'
+    mode: '0444'
+    owner: postgres
+    group: postgres
+  loop:
+    - testdb.sql
+
+- name: "Ensure test DB"
+  become: yes
+  become_user: postgres
+  community.postgresql.postgresql_db:
+    name: dummytestdb
+
+- name: "Ensure content for test DB"
+  become: yes
+  become_user: postgres
+  community.postgresql.postgresql_db:
+    name: dummytestdb
+    state: restore
+    target: /tmp/testdb.sql

roles/restore_maria/files/restore.sh

@@ -12,6 +12,7 @@ DATE=$(date +%F)
 
 systemctl stop mariadb
 
+rm -rf ${DATADIR}_moved
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}
 

roles/restore_postgres/files/restore.sh

@@ -13,6 +13,8 @@ PG_GROUP=postgres
 
 systemctl stop postgresql
 
+rm -rf ${DATADIR}_moved
+
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}