From 43da648df685a6785091e793cd9b58127861aacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6rz=2C=20Friedrich?= Date: Wed, 30 Mar 2022 15:57:34 +0000 Subject: [PATCH] DEV-389: added gpg-decryption for backup --- create-remote-database-backup.yml | 5 +- group_vars/all/plain.yml | 18 ++---- group_vars/{bastelserver => backup}/plain.yml | 7 ++- hcloud_firewall.yml | 6 -- restore-remote-database-backup.yml | 55 ++++++++++++++++--- .../defaults/main.yml | 0 .../files/pull_remote_backups.sh | 1 - .../files/push_backups_to_restore_server.sh | 4 +- .../{storage_server => backup}/tasks/main.yml | 7 ++- roles/management/tasks/configurations.yml | 1 + roles/restore_maria/files/restore.sh | 6 +- roles/restore_maria/tasks/main.yml | 17 ++++++ roles/restore_postgres/files/restore.sh | 6 +- roles/restore_postgres/tasks/main.yml | 17 ++++++ smardigo.yml | 3 + stage-dev | 5 +- stage-prodnso | 4 ++ stage-qa | 4 ++ storage_server.yml | 4 -- users/backuphamster/ssh.pub | 1 + 20 files changed, 125 insertions(+), 46 deletions(-) rename group_vars/{bastelserver => backup}/plain.yml (62%) rename roles/{storage_server => backup}/defaults/main.yml (100%) rename roles/{storage_server => backup}/files/pull_remote_backups.sh (99%) rename roles/{storage_server => backup}/files/push_backups_to_restore_server.sh (89%) rename roles/{storage_server => backup}/tasks/main.yml (83%) delete mode 100644 storage_server.yml create mode 100644 users/backuphamster/ssh.pub diff --git a/create-remote-database-backup.yml b/create-remote-database-backup.yml index 35e889d..2cce8fa 100644 --- a/create-remote-database-backup.yml +++ b/create-remote-database-backup.yml @@ -51,7 +51,7 @@ changed_when: False - name: "Add 'storage' servers to hosts if necessary" add_host: - name: "{{ stage }}-fgrz-01" + name: "{{ stage }}-backup-01" groups: - "stage_{{ stage }}" - storage @@ -92,7 +92,8 @@ become: yes become_user: '{{ storageserver_system_user }}' vars: - database_server_ip: "{{ stage }}-{{ database_engine }}-01.{{ domain }}" + # should work with non-fqdn due to existing entry in /etc/hosts + database_server_ip: "{{ stage }}-{{ database_engine }}-01" shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' when: - inventory_hostname in groups['storage'] diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index 5cf1816..b421a0e 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -128,9 +128,7 @@ ip_whitelist: - "46.245.219.98/32" # netgo borken - "{{ shared_service_network }}" -# for test purpose DEV-361 -# currently (2022.03.18) set to IP of hetzner VM -gitlab_storage_server: 167.235.18.147/32 +offsite_storage_server_ip: 142.132.155.83/32 docker_owner: "{{ admin_user }}" docker_group: "{{ admin_user }}" @@ -382,7 +380,7 @@ hcloud_firewall_objects_awx: hcloud_firewall_objects_backup: - - name: "{{ stage }}-database-backup-ssh-access" + name: "{{ stage }}-backup-ssh-access" state: present rules: - @@ -390,19 +388,11 @@ hcloud_firewall_objects_backup: protocol: tcp port: '22' source_ips: - - "{{ gitlab_storage_server }}" + - "{{ offsite_storage_server_ip }}" destination_ips: [] description: null apply_to: - type: label_selector label_selector: - selector: 'service=postgres' - - - type: label_selector - label_selector: - selector: 'service=maria' - - - type: label_selector - label_selector: - selector: 'service=restore' + selector: 'service=backup' diff --git a/group_vars/bastelserver/plain.yml b/group_vars/backup/plain.yml similarity index 62% rename from group_vars/bastelserver/plain.yml rename to group_vars/backup/plain.yml index 5c45e4f..a2d9d03 100644 --- a/group_vars/bastelserver/plain.yml +++ b/group_vars/backup/plain.yml @@ -1,9 +1,12 @@ --- #TODO needs to be removed after story DEV-361 is finished hetzner_server_type: "{{ hetzner_server_type_bastelserver | default('cx21') }}" -hetzner_server_labels: "stage={{ stage }} service=bastelserver" +hetzner_server_labels: "stage={{ stage }} service=backup" docker_enabled: false traefik_enabled: false filebeat_enabled: false -node_exporter_enabled: false +node_exporter_enabled: true + +custom_plattform_users: + - backuphamster diff --git a/hcloud_firewall.yml b/hcloud_firewall.yml index 94600bf..bd2ee5c 100644 --- a/hcloud_firewall.yml +++ b/hcloud_firewall.yml @@ -80,9 +80,6 @@ loop: "{{ hcloud_firewall_objects_awx }}" loop_control: loop_var: firewall_object - when: - - awx_related is defined - - awx_related - name: "Setup hcloud firewalls for database backup stuff..." include_role: @@ -91,6 +88,3 @@ loop: "{{ hcloud_firewall_objects_backup }}" loop_control: loop_var: firewall_object - when: - - backup_related is defined - - backup_related diff --git a/restore-remote-database-backup.yml b/restore-remote-database-backup.yml index 347781e..055775e 100644 --- a/restore-remote-database-backup.yml +++ b/restore-remote-database-backup.yml @@ -49,12 +49,12 @@ - "stage_{{ stage }}" - 'restore' changed_when: False - - name: "Add 'storage' servers to hosts if necessary" + - name: "Add 'backup' servers to hosts if necessary" add_host: - name: "{{ stage }}-fgrz-01" + name: "{{ stage }}-backup-01" groups: - "stage_{{ stage }}" - - storage + - backup changed_when: False ############################################################# @@ -98,24 +98,61 @@ - role: restore_{{ database_engine }} ############################################################# -# Syncing backups from storage server to restore server +# add restore specific firewall rule ############################################################# -- hosts: "storage" +- hosts: "{{ stage }}-virtual-host-to-read-groups-vars" + serial: "{{ serial_number | default(1) }}" + gather_facts: false + connection: local + vars: + hcloud_firewall_objects_backup: + - + name: "{{ stage }}-restore-ssh-access" + state: present + rules: + - + direction: in + protocol: tcp + port: '22' + source_ips: + - "{{ lookup('community.general.dig', groups['backup'][0] + '.' + domain ) }}/32" + destination_ips: [] + description: null + apply_to: + - + type: label_selector + label_selector: + selector: 'service=restore' + + tasks: + - name: "Add hcloud firewall rule(s)" + include_role: + name: hcloud + tasks_from: configure-firewall2 + loop: "{{ hcloud_firewall_objects_backup }}" + loop_control: + loop_var: firewall_object + +############################################################# +# Syncing backups from backup server to restore server +############################################################# + +- hosts: "backup" serial: "{{ serial_number | default(5) }}" gather_facts: false vars: - storageserver_system_user: 'backuphamster' + backupserver_system_user: 'backuphamster' ansible_ssh_host: "{{ stage_server_domain }}" tasks: # I could not get it up and running with module # to sync data from remote server A to remote server B - name: "Syncing remote backups" become: yes - become_user: '{{ storageserver_system_user }}' + become_user: '{{ backupserver_system_user }}' vars: - database_server_ip: "{{ stage }}-restore-{{ database_engine }}-01.{{ domain }}" - shell: '/home/{{ storageserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' + database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}" + shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' ############################################################# # Restoring from backup diff --git a/roles/storage_server/defaults/main.yml b/roles/backup/defaults/main.yml similarity index 100% rename from roles/storage_server/defaults/main.yml rename to roles/backup/defaults/main.yml diff --git a/roles/storage_server/files/pull_remote_backups.sh b/roles/backup/files/pull_remote_backups.sh similarity index 99% rename from roles/storage_server/files/pull_remote_backups.sh rename to roles/backup/files/pull_remote_backups.sh index 20f56fc..746377c 100644 --- a/roles/storage_server/files/pull_remote_backups.sh +++ b/roles/backup/files/pull_remote_backups.sh @@ -11,4 +11,3 @@ DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/ mkdir -p ${DEST_DIR} rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/ - diff --git a/roles/storage_server/files/push_backups_to_restore_server.sh b/roles/backup/files/push_backups_to_restore_server.sh similarity index 89% rename from roles/storage_server/files/push_backups_to_restore_server.sh rename to roles/backup/files/push_backups_to_restore_server.sh index cdded15..9dd5010 100644 --- a/roles/storage_server/files/push_backups_to_restore_server.sh +++ b/roles/backup/files/push_backups_to_restore_server.sh @@ -12,14 +12,14 @@ DATABASE_ENGINE=$3 DATE=$(date +%F) LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}" -BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz | head -n 1) +BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1) REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}" DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/" SSH_OPTIONS='-o StrictHostKeyChecking=no' -# needed due to unknown rsycn option --mkpath in rsycn version 3.1.3 +# needed due to unknown rsync option --mkpath in rsync version 3.1.3 ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}" rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR} diff --git a/roles/storage_server/tasks/main.yml b/roles/backup/tasks/main.yml similarity index 83% rename from roles/storage_server/tasks/main.yml rename to roles/backup/tasks/main.yml index 7de464c..d81b2cd 100644 --- a/roles/storage_server/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -29,8 +29,11 @@ - name: "Providing rsync script" become: yes copy: - src: pull_remote_backups.sh - dest: '/home/{{ system_user }}/pull_remote_backups.sh' + src: '{{ item }}' + dest: '/home/{{ system_user }}/{{ item }}' mode: '0755' owner: '{{ system_user }}' group: '{{ system_user }}' + with_items: + - pull_remote_backups.sh + - push_backups_to_restore_server.sh diff --git a/roles/management/tasks/configurations.yml b/roles/management/tasks/configurations.yml index 8146ec0..19e9080 100644 --- a/roles/management/tasks/configurations.yml +++ b/roles/management/tasks/configurations.yml @@ -35,6 +35,7 @@ path: "./smardigo/{{ item }}" dest: "{{ temp.path }}/{{ item }}.zip" format: zip + mode: '0644' with_items: "{{ connect_configurations }}" tags: - update_configurations diff --git a/roles/restore_maria/files/restore.sh b/roles/restore_maria/files/restore.sh index 4890a8d..dfeec49 100644 --- a/roles/restore_maria/files/restore.sh +++ b/roles/restore_maria/files/restore.sh @@ -15,7 +15,11 @@ systemctl stop mariadb mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR} -cat /home/backupuser/backups/${STAGE}/maria/${DATE}/mariabackupstream_${DATE}_*.gz | gunzip | mbstream --directory ${DATADIR} -x --parallel=2 +LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria" +BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1) + +# --batch => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" << +gpg --batch --decrypt $BACKUP_FILE_ENCRYPTED | gunzip | mbstream --directory ${DATADIR} -x --parallel=2 mariabackup --prepare --target-dir=${DATADIR} diff --git a/roles/restore_maria/tasks/main.yml b/roles/restore_maria/tasks/main.yml index 795fc47..a2b1953 100644 --- a/roles/restore_maria/tasks/main.yml +++ b/roles/restore_maria/tasks/main.yml @@ -15,3 +15,20 @@ mode: '0750' owner: root group: root + +- name: "Create file for gpg secret key" + become: yes + copy: + dest: '/root/gpg_private_key' + mode: '0600' + owner: 'root' + group: 'root' + content: | + {{ gpg_key_smardigo_automation__private }} + +- name: "Import private gpg key" # noqa command-instead-of-shell + become: yes + shell: 'gpg --import /root/gpg_private_key' + register: gpg_import + changed_when: + - gpg_import.rc != '0' diff --git a/roles/restore_postgres/files/restore.sh b/roles/restore_postgres/files/restore.sh index b2cd735..44b5edb 100644 --- a/roles/restore_postgres/files/restore.sh +++ b/roles/restore_postgres/files/restore.sh @@ -16,7 +16,11 @@ systemctl stop postgresql mv ${DATADIR} ${DATADIR}_moved mkdir -p ${DATADIR} -tar -ixzf /home/backupuser/backups/${STAGE}/postgres/${DATE}/basebackup_${DATE}_*.tar.gz -C ${DATADIR} +LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres" +BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1) + +# --batch => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" << +gpg --batch --decrypt $BACKUP_FILE_ENCRYPTED | tar -xz -C ${DATADIR} chmod 0700 ${DATADIR} chown -R ${PG_USER}:${PG_GROUP} ${DATADIR} diff --git a/roles/restore_postgres/tasks/main.yml b/roles/restore_postgres/tasks/main.yml index b14ddea..5455052 100644 --- a/roles/restore_postgres/tasks/main.yml +++ b/roles/restore_postgres/tasks/main.yml @@ -10,3 +10,20 @@ mode: 0754 owner: root group: root + +- name: "Create file for gpg secret key" + become: yes + copy: + dest: '/root/gpg_private_key' + mode: '0600' + owner: 'root' + group: 'root' + content: | + {{ gpg_key_smardigo_automation__private }} + +- name: "Import private gpg key" # noqa command-instead-of-shell + become: yes + shell: 'gpg --import /root/gpg_private_key' + register: gpg_import + changed_when: + - gpg_import.rc != '0' diff --git a/smardigo.yml b/smardigo.yml index 32d0dc8..8b798bc 100644 --- a/smardigo.yml +++ b/smardigo.yml @@ -71,3 +71,6 @@ - role: pdns when: "'pdns' in group_names" + + - role: backup + when: "'backup' in group_names" diff --git a/stage-dev b/stage-dev index d1d9025..99d0cc6 100644 --- a/stage-dev +++ b/stage-dev @@ -1,5 +1,6 @@ -[bastelserver] +[backup] dev-fgrz-01 +dev-backup-01 [connect] dev-management-01 @@ -76,7 +77,7 @@ kube_control_plane kube_node [stage_dev:children] -bastelserver +backup connect elastic pdns diff --git a/stage-prodnso b/stage-prodnso index 7d76e57..546ebf1 100644 --- a/stage-prodnso +++ b/stage-prodnso @@ -1,3 +1,6 @@ +[backup] +prodnso-backup-01 + [connect] prodnso-management-01 @@ -72,6 +75,7 @@ kube_control_plane kube_node [stage_prodnso:children] +backup connect elastic pdns diff --git a/stage-qa b/stage-qa index 8ba0f14..633f3d5 100644 --- a/stage-qa +++ b/stage-qa @@ -1,3 +1,6 @@ +[backup] +qa-backup-01 + [connect] qa-management-01 @@ -72,6 +75,7 @@ kube_control_plane kube_node [stage_qa:children] +backup connect elastic pdns diff --git a/storage_server.yml b/storage_server.yml deleted file mode 100644 index 5279156..0000000 --- a/storage_server.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- hosts: bastelserver - roles: - - role: storage_server diff --git a/users/backuphamster/ssh.pub b/users/backuphamster/ssh.pub new file mode 100644 index 0000000..70139a1 --- /dev/null +++ b/users/backuphamster/ssh.pub @@ -0,0 +1 @@ +ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFRlmqgkIJxBC45cbVX25P1Uam/+Ct7XFvgMm60TDOWkQiTuVp5vd1sHq2HCRRfGxPrsKmwSQS5wMYIjeiclTag= friedrich@friedrich-HP-ZBook