From 6862f58b7888b19057f7ccc9789a0097dde5e0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20H=C3=A4hnel?= Date: Thu, 22 Jun 2023 14:57:36 +0000 Subject: [PATCH] Feature/dev 1110 Extend backup process to support multiple db servers per stage --- create-remote-database-backup.yml | 63 +++++----- group_vars/all/database.yml | 4 +- group_vars/all/versions.yml | 2 +- group_vars/postgres01/plain.yml | 14 +-- group_vars/postgres02/plain.yml | 14 +-- inventory_plugins/netgo-hcloud.py | 4 +- .../smardigo_user_token.cpython-311.pyc | Bin 0 -> 1345 bytes restore-remote-database-backup.yml | 114 ++++++++---------- roles/backup/files/pull_remote_backups.sh | 14 +-- .../files/push_backups_to_restore_server.sh | 19 +-- roles/lvm_with_hetzner_volumes/tasks/main.yml | 4 +- roles/postgres/tasks/master-requirements.yml | 10 +- roles/restore_maria/files/restore.sh | 4 +- roles/restore_maria/tasks/_restore_test.yml | 25 ++++ roles/restore_postgres/files/restore.sh | 3 +- .../restore_postgres/tasks/_restore_test.yml | 24 ++++ tasks/autodiscover_pre_tasks.yml | 17 ++- 17 files changed, 178 insertions(+), 157 deletions(-) create mode 100644 library/__pycache__/smardigo_user_token.cpython-311.pyc create mode 100644 roles/restore_maria/tasks/_restore_test.yml create mode 100644 roles/restore_postgres/tasks/_restore_test.yml diff --git a/create-remote-database-backup.yml b/create-remote-database-backup.yml index 945da45..0e6fc8d 100644 --- a/create-remote-database-backup.yml +++ b/create-remote-database-backup.yml @@ -1,5 +1,4 @@ --- - # ****** ** ****** ** ** ** ** ******* # # /*////** **** **////** /** ** /** /** /**////** # # /* /** **//** ** // /** ** /** /** /** /** # @@ -35,12 +34,12 @@ gather_facts: false tasks: -# add virtual server to load stage specific variables as context + # add virtual server to load stage specific variables as context - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts" add_host: name: "{{ stage }}-virtual-host-to-read-groups-vars" groups: - - "stage_{{ stage }}" + - "stage_{{ stage }}" changed_when: False - hosts: "{{ stage }}-virtual-host-to-read-groups-vars" @@ -55,31 +54,32 @@ tags: - always - tasks: - - name: "Add {{ database_engine }} servers to hosts if necessary" - add_host: - name: "{{ shared_service_postgres_secondary }}" - groups: - - "stage_{{ stage }}" - - '{{ database_engine }}' - when: - - database_engine in ['postgres'] + - name: "Import autodiscover pre-tasks" + import_tasks: tasks/autodiscover_pre_tasks.yml + become: false + tags: + - always + tasks: - name: "Add {{ database_engine }} servers to hosts if necessary" add_host: - name: "{{ shared_service_maria_primary }}" + name: "{{ item.name }}" groups: - - "stage_{{ stage }}" - - '{{ database_engine }}' + - "stage_{{ stage }}" + - "{{ database_engine }}" when: - - database_engine in ['maria'] + - (database_engine == 'postgres' and item.service == 'postgres' and (item.role | default('')) == 'slave') + or (database_engine == 'maria' and item.service == 'maria') + loop: "{{ stage_server_infos }}" - - name: "Add 'storage' servers to hosts if necessary" + - name: "Add 'backup' servers to hosts if necessary" add_host: name: "{{ stage }}-backup-01" groups: - - "stage_{{ stage }}" - - "storage" + - "stage_{{ stage }}" + - "backup" + when: + - "'postgres' in groups or 'maria' in groups" ############################################################## ## Creating remote database backups for created inventory @@ -90,45 +90,44 @@ gather_facts: false vars: ansible_ssh_host: "{{ stage_server_domain }}" - current_date_time: '{{ get_current_date_time }}' + current_date_time: "{{ get_current_date_time }}" tasks: - name: "Trigger backup mechanism" include_role: - name: '{{ database_engine }}' + name: "{{ database_engine }}" tasks_from: _create_backup ############################################################# -# Syncing remote database backups to storage server +# Syncing remote database backups to backup server ############################################################# -- hosts: "postgres:maria:storage" +- hosts: "postgres:maria:backup" serial: "{{ serial_number | default(5) }}" gather_facts: false vars: ansible_ssh_host: "{{ stage_server_domain }}" - storageserver_system_user: 'backuphamster' + backup_server_system_user: "backuphamster" tasks: # I could not get it up and running with module # to sync data from remote server A to remote server B - name: "Syncing remote backups" become: yes - become_user: '{{ storageserver_system_user }}' - vars: - # should work with non-fqdn due to existing entry in /etc/hosts - database_server_ip: "{{ shared_service_postgres_secondary if database_engine == 'postgres' else shared_service_maria_primary }}" - shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' + become_user: "{{ backup_server_system_user }}" + shell: "/home/{{ backup_server_system_user }}/pull_remote_backups.sh {{ item }} {{ stage }} {{ database_engine }}" + with_items: "{{ (groups['postgres'] | default([])) + (groups['maria'] | default([])) }}" when: - - inventory_hostname in groups['storage'] + - inventory_hostname in groups['backup'] - name: "Cleanup remote backup dirs: {{ database_engine }}" become: yes file: - path: '{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}' + path: "{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}" state: absent when: - - not inventory_hostname in groups['storage'] + - not inventory_hostname in groups['backup'] + - inventory_hostname in groups [database_engine] ############################################################# # Sending smardigo management message to process diff --git a/group_vars/all/database.yml b/group_vars/all/database.yml index 25dc924..5cda673 100644 --- a/group_vars/all/database.yml +++ b/group_vars/all/database.yml @@ -5,13 +5,13 @@ shared_service_postgres_primary: "{{ stage }}-postgres-01" shared_service_postgres_secondary: "{{ stage }}-postgres-02" shared_service_pg_master_ip: "{{ stage_server_infos - | selectattr('name', 'match', shared_service_postgres_primary ) + | selectattr('name', '==', shared_service_postgres_primary ) | map(attribute='private_ip') | list | first | default('-') }}" shared_service_pg_slave_ip: "{{ stage_server_infos - | selectattr('name', 'match', shared_service_postgres_secondary ) + | selectattr('name', '==', shared_service_postgres_secondary ) | map(attribute='private_ip') | list | first diff --git a/group_vars/all/versions.yml b/group_vars/all/versions.yml index f2eaf0f..63c32b9 100644 --- a/group_vars/all/versions.yml +++ b/group_vars/all/versions.yml @@ -26,6 +26,6 @@ traefik_version: "v2.10.3" connect_version: "10.5" iam_version: "10.5" -ansible_minimal_version: "2.12.0" +ansible_minimal_version: "2.12.0" wireguard_version: latest diff --git a/group_vars/postgres01/plain.yml b/group_vars/postgres01/plain.yml index 75d4ee6..20bd39a 100644 --- a/group_vars/postgres01/plain.yml +++ b/group_vars/postgres01/plain.yml @@ -1,13 +1,3 @@ --- -shared_service_pg_master_ip: "{{ stage_server_infos - | selectattr('name', 'match', stage + '-postgres-01' ) - | map(attribute='private_ip') - | list - | first - | default('-') }}" -shared_service_pg_slave_ip: "{{ stage_server_infos - | selectattr('name', 'match', stage + '-postgres-02' ) - | map(attribute='private_ip') - | list - | first - | default('-') }}" +shared_service_postgres_primary: "{{ stage }}-postgres-01" +shared_service_postgres_secondary: "{{ stage }}-postgres-02" diff --git a/group_vars/postgres02/plain.yml b/group_vars/postgres02/plain.yml index 17dee4e..dadea24 100644 --- a/group_vars/postgres02/plain.yml +++ b/group_vars/postgres02/plain.yml @@ -1,13 +1,3 @@ --- -shared_service_pg_master_ip: "{{ stage_server_infos - | selectattr('name', 'match', stage + '-postgres02-01' ) - | map(attribute='private_ip') - | list - | first - | default('-') }}" -shared_service_pg_slave_ip: "{{ stage_server_infos - | selectattr('name', 'match', stage + '-postgres02-02' ) - | map(attribute='private_ip') - | list - | first - | default('-') }}" +shared_service_postgres_primary: "{{ stage }}-postgres02-01" +shared_service_postgres_secondary: "{{ stage }}-postgres02-02" diff --git a/inventory_plugins/netgo-hcloud.py b/inventory_plugins/netgo-hcloud.py index b87270d..b9dc750 100644 --- a/inventory_plugins/netgo-hcloud.py +++ b/inventory_plugins/netgo-hcloud.py @@ -222,8 +222,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): serverId = server["id"] serverName = server["name"] serverLabels = server["labels"] - serverStage = serverLabels["stage"] - serverService = serverLabels["service"] + serverStage = serverLabels.get("stage", "this_stage_does_not_exist") + serverService = serverLabels.get("service", "this_group_does_not_exist") serverPublicIp = server["public_net"]["ipv4"]["ip"] serverPrivateIp = '-' diff --git a/library/__pycache__/smardigo_user_token.cpython-311.pyc b/library/__pycache__/smardigo_user_token.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec79a79021c1627f6a5e8c0a7c395dab6156a756 GIT binary patch literal 1345 zcmaJ=&2Jk;6rbH4ukEcjE#QVE99S%bY9z;@dVo+My-HOy^Yx07$PgV08=RWa^a?8CA3w-}M4B$Un< z6gKJ2<`g1OR9Ns3=<*B%fS$npNS^Nakvu?w1eaqvKqsj~I3G`K11X3zjHxquMnu#D zsWZJH>G_EZ>VrlyvHXum_8$+Tkt-*;Lkx9bjc6brNROwdi8bDdejuOB975&eX?1vh zw1tXSH)=xR0V~QPP3W!#I0+o=cPSWDo~2%_y#r`+_3?X}8n8uUHNO4g`o*|8hUV*BOijNf8+Utrg= zJvIlqXX7FdKnzi&qE}x(+74fQ8~6fL8ou-naFkMJ)ChqX*8bAa%qvH;p}usgFP-a4 z!`iR8tKWY5P^EZ}H=a!%5Pv@#n@BKIkx9^;Pd^ha)5kdoLO@>Whm;XrQ zpNSk+f5zHRIDdxo59>$QL!3Xw<#SvPCwvL5A(1i7xMS1YD5iyeYjbsby}D6fso&n* z5Si-dE9;-FRksB0_<&J@^uid3#SxZfFR&fH*s!?WO#XJ9G5&r{^9f?{RI-;rJj;AA z@?U1J!7=^-_%4V@kt8Wn@e6pUM_M+AhnZ*=N!LS^|2JQt8{vc_d_`J{&{QlSocpGK crp$-R`~}VoZ+%_6UmGUBNI`f3%t;@A17w435&!@I literal 0 HcmV?d00001 diff --git a/restore-remote-database-backup.yml b/restore-remote-database-backup.yml index 4f016cc..ada33ac 100644 --- a/restore-remote-database-backup.yml +++ b/restore-remote-database-backup.yml @@ -1,5 +1,4 @@ --- - # restores remote database backup # - postgres # - executed on stage specific server: {{ shared_service_postgres_primary }}-restore @@ -40,32 +39,46 @@ - "stage_{{ stage }}" changed_when: False +- hosts: "{{ stage }}-virtual-host-to-read-groups-vars" + serial: "{{ serial_number | default(1) }}" + gather_facts: false + connection: local + + pre_tasks: + - name: "Import autodiscover pre-tasks" + import_tasks: tasks/autodiscover_pre_tasks.yml + become: false + tags: + - always + tasks: - - name: "Add {{ database_engine }}-restore servers to hosts if necessary" - add_host: - name: "{{ shared_service_postgres_secondary }}-restore" - groups: - - "stage_{{ stage }}" - - 'restore' + - name: "Get all {{ database_engine }} servers in stage {{ stage }}" + set_fact: + database_servers: "{{ database_servers | default([]) + [item.name] }}" when: - - database_engine in ['postgres'] + - (database_engine == 'postgres' and item.service == 'postgres' and (item.role | default('')) == 'slave') + or (database_engine == 'maria' and item.service == 'maria') + loop: "{{ stage_server_infos }}" - - name: "Add {{ database_engine }}-restore servers to hosts if necessary" + - name: "Add {{ database_engine }} restore server to hosts if necessary" add_host: - name: "{{ shared_service_maria_primary }}-restore" + name: "{{ stage }}-{{ database_engine }}-restore" groups: - - "stage_{{ stage }}" - - 'restore' + - "stage_{{ stage }}" + - "restore" when: - - database_engine in ['maria'] - + - (database_engine == 'postgres') + or (database_engine == 'maria') + - name: "Add 'backup' servers to hosts if necessary" add_host: name: "{{ stage }}-backup-01" groups: - - "stage_{{ stage }}" - - backup - changed_when: False + - "stage_{{ stage }}" + - "backup" + when: + - (database_engine == 'postgres') + or (database_engine == 'maria') ############################################################# # Create restore server(s) @@ -165,17 +178,23 @@ serial: "{{ serial_number | default(5) }}" gather_facts: false vars: - backupserver_system_user: 'backuphamster' + backup_server_system_user: 'backuphamster' ansible_ssh_host: "{{ stage_server_domain }}" + database_servers: "{{ hostvars[stage + '-virtual-host-to-read-groups-vars'].database_servers }}" tasks: # I could not get it up and running with module # to sync data from remote server A to remote server B - name: "Syncing remote backups" become: yes - become_user: '{{ backupserver_system_user }}' + become_user: '{{ backup_server_system_user }}' vars: - database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}" - shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' + restore_server: "{{ groups['restore'][0] }}.{{ domain }}" + shell: '/home/{{ backup_server_system_user }}/push_backups_to_restore_server.sh {{ restore_server }} {{ database_server }} {{ stage }} {{ database_engine }}' + loop: "{{ database_servers }}" + loop_control: + loop_var: database_server + when: + - inventory_hostname in groups['backup'] ############################################################# # Restoring from backup @@ -186,52 +205,15 @@ gather_facts: false vars: ansible_ssh_host: "{{ stage_server_domain }}" + database_servers: "{{ hostvars[stage + '-virtual-host-to-read-groups-vars'].database_servers }}" tasks: - - name: "Triggering restore" - become: yes - shell: '/root/restore.sh {{ stage }}' - - - name: "Check for test data on postgres" - block: - - - name: "Querying postgres ..." - become: yes - become_user: postgres - community.postgresql.postgresql_query: - db: dummytestdb - query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s - named_args: - quote_val: 'Shall we play' - register: query_output - - - assert: - that: - - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1' - - when: - - database_engine == 'postgres' - - - name: "Check for test data on mariadb" - block: - - - name: "Querying mariadb ..." - become: yes - become_user: root - community.mysql.mysql_query: - login_unix_socket: /run/mysqld/mysqld.sock - login_db: dummytestdb - query: SELECT movie FROM movie_quotes WHERE quote = %s - positional_args: - - 'Shall we play' - register: query_output - - - assert: - that: - - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1' - - when: - - database_engine == 'maria' - + - name: "Start restore test for '{{ database_engine }}' backup from server '{{ database_server }}'" + include_role: + name: "restore_{{ database_engine }}" + tasks_from: _restore_test + loop: "{{ database_servers }}" + loop_control: + loop_var: database_server ############################################################# # Deleting volumes for created inventory diff --git a/roles/backup/files/pull_remote_backups.sh b/roles/backup/files/pull_remote_backups.sh index 0ffb791..8c6bef5 100644 --- a/roles/backup/files/pull_remote_backups.sh +++ b/roles/backup/files/pull_remote_backups.sh @@ -10,22 +10,22 @@ set -euo pipefail DATE=$(date +%F) DATE_TIME=$(date +%F_%H:%M) REMOTE_SYSTEM_USER=backupuser -DATABASE_SERVER_IP=$1 +DATABASE_SERVER=$1 STAGE=$2 DATABASE_ENGINE=$3 -DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE} +DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER} METRICS_FILE=${HOME}/backup_status_${DATABASE_ENGINE}.prom LOG_FILE=${DEST_DIR}/backup_${DATE_TIME}.log +# Create backup directory ${DEST_DIR} if not exist +mkdir -p ${DEST_DIR} + # Redirect stderr to stdout and save everything to log file exec > ${LOG_FILE} 2>&1 # Log backup sync start time echo "----- Start backup Sync - ${DATE_TIME} -----" -# Create backup directory ${DEST_DIR} if not exist -mkdir -p ${DEST_DIR} - # Remove files oder than 48h in ${DEST_DIR} find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR" [ "$?" != "0" ] && exit 1 @@ -33,8 +33,8 @@ find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR" echo "Removing logfiles older than 7d ..." find $DEST_DIR -type f -mtime +7 -name "backup_*.log" -print0 | xargs -I OLD_FILES -0 rm -rf "OLD_FILES" -# Start rsync job from ${DATABASE_SERVER_IP} to ${DEST_DIR}/ -rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/ +# Start rsync job from ${DATABASE_SERVER} to ${DEST_DIR}/ +rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/ [ "$?" -eq "0" ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1" BACKUP_STATUS_FILE=$(ls -t1 ${DEST_DIR}/${DATE}/backup_finished_${DATE}_* | head -n1) diff --git a/roles/backup/files/push_backups_to_restore_server.sh b/roles/backup/files/push_backups_to_restore_server.sh index 1a0ec16..e654768 100644 --- a/roles/backup/files/push_backups_to_restore_server.sh +++ b/roles/backup/files/push_backups_to_restore_server.sh @@ -4,17 +4,18 @@ # REMOTE_SYSTEM_USER=backupuser -DATABASE_SERVER_IP=$1 -STAGE=$2 -DATABASE_ENGINE=$3 +RESTORE_SERVER=$1 +DATABASE_SERVER=$2 +STAGE=$3 +DATABASE_ENGINE=$4 # currently it defaults to todays date DATE=$(date +%F) -LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}" +LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}" BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1) -REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}" +REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}" DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/" if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then @@ -22,16 +23,16 @@ if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then fi # avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process -ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${DATABASE_SERVER_IP} +ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${RESTORE_SERVER} SSH_OPTIONS='-o StrictHostKeyChecking=no' # needed due to unknown rsync option --mkpath in rsync version 3.1.3 -ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}" +ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "mkdir -p ${DEST_DIR}" -rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR} +rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER}:${DEST_DIR} BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}') -ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}" +ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}" diff --git a/roles/lvm_with_hetzner_volumes/tasks/main.yml b/roles/lvm_with_hetzner_volumes/tasks/main.yml index cd3c4fc..284c65e 100644 --- a/roles/lvm_with_hetzner_volumes/tasks/main.yml +++ b/roles/lvm_with_hetzner_volumes/tasks/main.yml @@ -69,12 +69,12 @@ when: create_vg.changed - name: "Format volume" - community.general.system.filesystem: + community.general.filesystem: fstype: ext4 dev: "/dev/{{ vg_name }}/{{ lv_name }}" - name: "Resize volume" # noqa no-handler - community.general.system.filesystem: + community.general.filesystem: fstype: ext4 dev: "/dev/{{ vg_name }}/{{ lv_name }}" resizefs: true diff --git a/roles/postgres/tasks/master-requirements.yml b/roles/postgres/tasks/master-requirements.yml index df5ed23..b34cee2 100644 --- a/roles/postgres/tasks/master-requirements.yml +++ b/roles/postgres/tasks/master-requirements.yml @@ -1,5 +1,4 @@ --- - - name: Check role exists # noqa no-changed-when become: true become_user: postgres @@ -181,9 +180,7 @@ register: cmd_ret changed_when: cmd_ret.rc != 0 -# only needed in case of install from scratch - name: "Ensure test db stuff" - when: postgres_ensure_testdb | default(False) block: - name: "Copy testdb.sql to ensure test DB" ansible.builtin.copy: @@ -195,6 +192,13 @@ loop: - testdb.sql + - name: "Drop test DB if exists" + become: true + become_user: postgres + community.postgresql.postgresql_db: + name: dummytestdb + state: absent + - name: "Ensure test DB" become: true become_user: postgres diff --git a/roles/restore_maria/files/restore.sh b/roles/restore_maria/files/restore.sh index 7372861..ff78b43 100644 --- a/roles/restore_maria/files/restore.sh +++ b/roles/restore_maria/files/restore.sh @@ -5,18 +5,18 @@ # STAGE=$1 +DATABASE_SERVER=$2 DATADIR='/var/lib/mysql' DATE=$(date +%F) - systemctl stop mariadb rm -rf ${DATADIR}_moved mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR} -LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria" +LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria/${DATABASE_SERVER}" BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1) # --batch => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" << diff --git a/roles/restore_maria/tasks/_restore_test.yml b/roles/restore_maria/tasks/_restore_test.yml new file mode 100644 index 0000000..620971b --- /dev/null +++ b/roles/restore_maria/tasks/_restore_test.yml @@ -0,0 +1,25 @@ +--- +- name: "Triggering restore" # noqa command-instead-of-shell no-changed-when + become: true + shell: "/root/restore.sh {{ stage }} {{ database_server }}" + +- name: "Check for test data on mariadb" + block: + - name: "Querying mariadb ..." + become: true + become_user: root + community.mysql.mysql_query: + login_unix_socket: /run/mysqld/mysqld.sock + login_db: dummytestdb + query: SELECT movie FROM movie_quotes WHERE quote = %s + positional_args: + - "Shall we play" + register: query_output + + - name: "Compare query result with given parameters ..." + assert: + that: + - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1' + + when: + - database_engine == 'maria' diff --git a/roles/restore_postgres/files/restore.sh b/roles/restore_postgres/files/restore.sh index 0f2278e..b707513 100644 --- a/roles/restore_postgres/files/restore.sh +++ b/roles/restore_postgres/files/restore.sh @@ -4,6 +4,7 @@ # STAGE=$1 +DATABASE_SERVER=$2 DATADIR='/var/lib/postgresql/13/main' DATE=$(date +%F) @@ -18,7 +19,7 @@ rm -rf ${DATADIR}_moved mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR} -LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres" +LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres/${DATABASE_SERVER}" BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1) # --batch => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" << diff --git a/roles/restore_postgres/tasks/_restore_test.yml b/roles/restore_postgres/tasks/_restore_test.yml new file mode 100644 index 0000000..dd85db0 --- /dev/null +++ b/roles/restore_postgres/tasks/_restore_test.yml @@ -0,0 +1,24 @@ +--- +- name: "Triggering restore" # noqa command-instead-of-shell no-changed-when + become: true + shell: "/root/restore.sh {{ stage }} {{ database_server }}" + +- name: "Check for test data on postgres" + block: + - name: "Querying postgres ..." + become: true + become_user: postgres + community.postgresql.postgresql_query: + db: dummytestdb + query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s + named_args: + quote_val: "Shall we play" + register: query_output + + - name: "Compare query result with given parameters ..." + assert: + that: + - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1' + + when: + - database_engine == 'postgres' diff --git a/tasks/autodiscover_pre_tasks.yml b/tasks/autodiscover_pre_tasks.yml index 59a709b..ab7f755 100644 --- a/tasks/autodiscover_pre_tasks.yml +++ b/tasks/autodiscover_pre_tasks.yml @@ -142,7 +142,6 @@ tags: - always - ############################################################# # Setting stage infos as facts ############################################################# @@ -151,21 +150,26 @@ - name: "Reading hetzner server infos for stage <{{ stage }}/{{ stage_kube }}>" set_fact: stage_server_infos: "\ - {% set list= [] %}\ + {% set stage_server_infos= [] %}\ {% for server in hetzner_servers %}\ {% if server.labels.manual is not defined %}\ + {% set stage_server_info={'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id} %}\ + {% for label in (server.labels | dict2items) %}\ + {% set x=stage_server_info.__setitem__(label.key,label.value) %}\ + {% endfor %}\ + + {% set x=stage_server_info.__setitem__('private_ip','-') %}\ {% if (server.private_net|length) > 0 %}\ {% for private_net in server.private_net %}\ {% if private_net.network == stage_network_id|int %}\ - {{ list.append({'service': server.labels.service | default(''), 'private_ip': private_net.ip, 'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id}) }}\ + {% set x=stage_server_info.__setitem__('private_ip',private_net.ip) %}\ {% endif %}\ {% endfor %}\ - {% else %}\ - {{ list.append({'service': server.labels.service | default(''), 'private_ip': '-', 'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id}) }}\ {% endif %}\ {% endif %}\ + {{ stage_server_infos.append(stage_server_info) }}\ {% endfor %}\ - {{ list|list }}" + {{ stage_server_infos | list }}" tags: - always @@ -202,5 +206,6 @@ - "ingess private ip: <{{ shared_service_kube_loadbalancer_public_ip }}>" - "logstash public ip: <{{ shared_service_loadbalancer_logstash_public_ip }}>" - "logstash private ip: <{{ shared_service_loadbalancer_logstash_private_ip }}>" + - "server info: <{{ stage_server_infos | selectattr('name', 'equalto', inventory_hostname) | default({}) }}>" tags: - always