From 6fbc3af3c47a0ba72687cfafbc91783a5f735760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6rz=2C=20Friedrich?= Date: Thu, 31 Mar 2022 19:18:41 +0000 Subject: [PATCH] DEV-374: implemented logical restore-test to check if restore was succussful --- restore-remote-database-backup.yml | 44 ++++++++++++++ roles/backup/files/pull_remote_backups.sh | 3 + roles/hcloud/tasks/_set_server_state.yml | 1 + roles/maria/files/testdb.sql | 52 +++++++++++++++++ roles/maria/handlers/main.yml | 6 ++ roles/maria/tasks/main.yml | 35 +++++++++--- roles/postgres/files/testdb.sql | 60 ++++++++++++++++++++ roles/postgres/tasks/base-requirements.yml | 7 ++- roles/postgres/tasks/master-requirements.yml | 24 ++++++++ roles/restore_maria/files/restore.sh | 1 + roles/restore_postgres/files/restore.sh | 2 + 11 files changed, 224 insertions(+), 11 deletions(-) create mode 100644 roles/maria/files/testdb.sql create mode 100644 roles/maria/handlers/main.yml create mode 100644 roles/postgres/files/testdb.sql diff --git a/restore-remote-database-backup.yml b/restore-remote-database-backup.yml index 055775e..7e831e1 100644 --- a/restore-remote-database-backup.yml +++ b/restore-remote-database-backup.yml @@ -168,6 +168,50 @@ become: yes shell: '/root/restore.sh {{ stage }}' + - name: "Check for test data on postgres" + block: + + - name: "Querying postgres ..." + become: yes + become_user: postgres + community.postgresql.postgresql_query: + db: dummytestdb + query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s + named_args: + quote_val: 'Shall we play' + register: query_output + + - assert: + that: + - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1' + + when: + - database_engine == 'postgres' + + - name: "Check for test data on mariadb" + block: + + - name: "Querying mariadb ..." + become: yes + become_user: root + community.mysql.mysql_query: + login_user: '{{ mysql_root_username }}' + login_password: "{{ mysql_root_password }}" + config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf" + login_db: dummytestdb + query: SELECT movie FROM movie_quotes WHERE quote = %s + positional_args: + - 'Shall we play' + register: query_output + + - assert: + that: + - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1' + + when: + - database_engine == 'maria' + + # TODO delete restore-server ############################################################# diff --git a/roles/backup/files/pull_remote_backups.sh b/roles/backup/files/pull_remote_backups.sh index 746377c..c48eec5 100644 --- a/roles/backup/files/pull_remote_backups.sh +++ b/roles/backup/files/pull_remote_backups.sh @@ -11,3 +11,6 @@ DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/ mkdir -p ${DEST_DIR} rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/ + +# remove files oder than XX in backup-DIR +find ${DEST_DIR} -ctime +7 -delete diff --git a/roles/hcloud/tasks/_set_server_state.yml b/roles/hcloud/tasks/_set_server_state.yml index 1bca4b4..fa0ecf3 100644 --- a/roles/hcloud/tasks/_set_server_state.yml +++ b/roles/hcloud/tasks/_set_server_state.yml @@ -29,6 +29,7 @@ poll: 15 when: - server_state == 'present' + - not hcloud_response.msg is defined # just catching hetzner error and trying to avoid manual interaction - name: "DEBUGGING Block for hetzner << message not found >> - error" diff --git a/roles/maria/files/testdb.sql b/roles/maria/files/testdb.sql new file mode 100644 index 0000000..b82c4a4 --- /dev/null +++ b/roles/maria/files/testdb.sql @@ -0,0 +1,52 @@ +-- MySQL dump 10.19 Distrib 10.3.34-MariaDB, for debian-linux-gnu (x86_64) +-- +-- Host: localhost Database: stararchitekt +-- ------------------------------------------------------ +-- Server version 10.3.34-MariaDB-0ubuntu0.20.04.1 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `movie_quotes` +-- + +DROP TABLE IF EXISTS `movie_quotes`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `movie_quotes` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `movie` varchar(255) DEFAULT NULL, + `quote` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `movie_quotes` +-- + +LOCK TABLES `movie_quotes` WRITE; +/*!40000 ALTER TABLE `movie_quotes` DISABLE KEYS */; +INSERT INTO `movie_quotes` VALUES (1,'wargames','Shall we play'); +/*!40000 ALTER TABLE `movie_quotes` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2022-03-31 9:46:16 diff --git a/roles/maria/handlers/main.yml b/roles/maria/handlers/main.yml new file mode 100644 index 0000000..3b25b50 --- /dev/null +++ b/roles/maria/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: "prometheus-mysqld-exporter restart" + service: + name: prometheus-mysqld-exporter + state: restarted + enabled: yes diff --git a/roles/maria/tasks/main.yml b/roles/maria/tasks/main.yml index 3f94559..2d5eadd 100644 --- a/roles/maria/tasks/main.yml +++ b/roles/maria/tasks/main.yml @@ -14,12 +14,6 @@ - python3-pymysql - prometheus-mysqld-exporter -- name: Stop prometheus-mysqld-exporter - service: - name: prometheus-mysqld-exporter - state: stopped - enabled: no - - name: Fix binding.. ansible.builtin.lineinfile: path: /etc/mysql/mariadb.conf.d/50-server.cnf @@ -77,6 +71,8 @@ priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT" config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf" login_password: "{{ mysql_root_password }}" + register: mysql_exporter_user_creds + notify: prometheus-mysqld-exporter restart - name: Ensure is prometheus-mysqld-exporter configured lineinfile: @@ -84,6 +80,7 @@ line: 'DATA_SOURCE_NAME="prometheus@unix(/run/mysqld/mysqld.sock)/"' path: /etc/default/prometheus-mysqld-exporter register: mysql_exporter_data_source + notify: prometheus-mysqld-exporter restart - name: Setup prometheus-mysqld-exporter interface bind lineinfile: @@ -91,8 +88,9 @@ regex: "^ARGS=" line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_maria }}'\"" register: mysql_exporter_args + notify: prometheus-mysqld-exporter restart -- name: Start prometheus-mysqld-exporter if needed +- name: "Ensure prometheus-mysqld-exporter is running" service: name: prometheus-mysqld-exporter state: started @@ -104,6 +102,25 @@ path: '{{ backup_directory }}' mode: 0755 +- name: "Copy testdb.sql to ensure test DB" + copy: + src: '{{ item }}' + dest: '/tmp/{{ item }}' + mode: '0444' + owner: root + group: root + loop: + - testdb.sql + +- name: "Ensure test DB" + community.mysql.mysql_db: + login_user: '{{ mysql_root_username }}' + login_password: "{{ mysql_root_password }}" + config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf" + name: dummytestdb + state: import + target: /tmp/testdb.sql + - name: "Block: gpg stuff" block: - name: Create temp dir @@ -112,13 +129,13 @@ suffix: gitcheckout path: /tmp register: tempdir - + - name: "Checkout repo for gpg communication-keys" ansible.builtin.git: repo: 'https://{{ gituser | default("gitea-admin") | urlencode }}:{{ gitea_admin_password | urlencode }}@{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys.git' dest: '{{ tempdir.path }}' version: master - + # there is no ansible gpg module already in place # linting violation needs to be whitelisted - name: "Importing stage specific automation gpg-key" # noqa command-instead-of-shell diff --git a/roles/postgres/files/testdb.sql b/roles/postgres/files/testdb.sql new file mode 100644 index 0000000..92e419d --- /dev/null +++ b/roles/postgres/files/testdb.sql @@ -0,0 +1,60 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 13.6 (Ubuntu 13.6-1.pgdg20.04+1) +-- Dumped by pg_dump version 13.6 (Ubuntu 13.6-1.pgdg20.04+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: movie_quotes; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.movie_quotes ( + movie character varying(255), + quote character varying(255) +); + + +ALTER TABLE public.movie_quotes OWNER TO postgres; + +-- +-- Data for Name: movie_quotes; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY public.movie_quotes (movie, quote) FROM stdin; +wargames Shall we play +\. + + +-- +-- PostgreSQL database dump complete +-- diff --git a/roles/postgres/tasks/base-requirements.yml b/roles/postgres/tasks/base-requirements.yml index b35583a..3464e7a 100644 --- a/roles/postgres/tasks/base-requirements.yml +++ b/roles/postgres/tasks/base-requirements.yml @@ -30,12 +30,15 @@ repo: deb http://apt.postgresql.org/pub/repos/apt {{ default_postgres_target_distribution }} main state: present -- name: "Ensuring postgres is installed" +- name: "Ensuring several packages being installed" apt: - name: "postgresql-{{ default_postgres_version }}" + name: "{{ item }}" update_cache: yes cache_valid_time: 900 state: present + loop: + - 'postgresql-{{ default_postgres_version }}' + - python3-psycopg2 - name: "Setup listen interfaces for postgresql instance" lineinfile: diff --git a/roles/postgres/tasks/master-requirements.yml b/roles/postgres/tasks/master-requirements.yml index 583ec93..411a839 100644 --- a/roles/postgres/tasks/master-requirements.yml +++ b/roles/postgres/tasks/master-requirements.yml @@ -157,3 +157,27 @@ shell: "/usr/bin/psql -Atc \"SELECT pg_create_physical_replication_slot('pgstandby1');\"" ignore_errors: yes # noqa ignore-errors when: replication_slot_check.stdout == "0" + +- name: "Copy testdb.sql to ensure test DB" + copy: + src: '{{ item }}' + dest: '/tmp/{{ item }}' + mode: '0444' + owner: postgres + group: postgres + loop: + - testdb.sql + +- name: "Ensure test DB" + become: yes + become_user: postgres + community.postgresql.postgresql_db: + name: dummytestdb + +- name: "Ensure content for test DB" + become: yes + become_user: postgres + community.postgresql.postgresql_db: + name: dummytestdb + state: restore + target: /tmp/testdb.sql diff --git a/roles/restore_maria/files/restore.sh b/roles/restore_maria/files/restore.sh index dfeec49..7372861 100644 --- a/roles/restore_maria/files/restore.sh +++ b/roles/restore_maria/files/restore.sh @@ -12,6 +12,7 @@ DATE=$(date +%F) systemctl stop mariadb +rm -rf ${DATADIR}_moved mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR} diff --git a/roles/restore_postgres/files/restore.sh b/roles/restore_postgres/files/restore.sh index 44b5edb..0f2278e 100644 --- a/roles/restore_postgres/files/restore.sh +++ b/roles/restore_postgres/files/restore.sh @@ -13,6 +13,8 @@ PG_GROUP=postgres systemctl stop postgresql +rm -rf ${DATADIR}_moved + mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR}