DEV-389: added gpg-decryption for backup

create-remote-database-backup.yml

@@ -51,7 +51,7 @@
       changed_when: False
     - name: "Add 'storage' servers to hosts if necessary"
       add_host:
-        name: "{{ stage }}-fgrz-01"
+        name: "{{ stage }}-backup-01"
         groups:
         - "stage_{{ stage }}"
         - storage
@@ -92,7 +92,8 @@
       become: yes
       become_user: '{{ storageserver_system_user }}'
       vars:
-        database_server_ip: "{{ stage }}-{{ database_engine }}-01.{{ domain }}"
+        # should work with non-fqdn due to existing entry in /etc/hosts
+        database_server_ip: "{{ stage }}-{{ database_engine }}-01"
       shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
       when:
         - inventory_hostname in groups['storage']

group_vars/all/plain.yml

@@ -128,9 +128,7 @@ ip_whitelist:
   - "46.245.219.98/32" # netgo borken
   - "{{ shared_service_network }}"
 
-# for test purpose DEV-361
+offsite_storage_server_ip: 142.132.155.83/32
-# currently (2022.03.18) set to IP of hetzner VM
-gitlab_storage_server: 167.235.18.147/32
 
 docker_owner: "{{ admin_user }}"
 docker_group: "{{ admin_user }}"
@@ -382,7 +380,7 @@ hcloud_firewall_objects_awx:
 
 hcloud_firewall_objects_backup:
    -
-     name: "{{ stage }}-database-backup-ssh-access"
+     name: "{{ stage }}-backup-ssh-access"
      state: present
      rules:
      -
@@ -390,19 +388,11 @@ hcloud_firewall_objects_backup:
         protocol: tcp
         port: '22'
         source_ips:
-          - "{{ gitlab_storage_server }}"
+          - "{{ offsite_storage_server_ip }}"
         destination_ips: []
         description: null
      apply_to:
      -
        type: label_selector
        label_selector:
-         selector: 'service=postgres'
+         selector: 'service=backup'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=maria'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=restore'

group_vars/backup/plain.yml

@@ -1,9 +1,12 @@
 ---
 #TODO needs to be removed after story DEV-361 is finished
 hetzner_server_type: "{{ hetzner_server_type_bastelserver | default('cx21') }}"
-hetzner_server_labels: "stage={{ stage }} service=bastelserver"
+hetzner_server_labels: "stage={{ stage }} service=backup"
 
 docker_enabled: false
 traefik_enabled: false
 filebeat_enabled: false
-node_exporter_enabled: false
+node_exporter_enabled: true
+
+custom_plattform_users:
+  - backuphamster

hcloud_firewall.yml

@@ -80,9 +80,6 @@
         loop: "{{ hcloud_firewall_objects_awx }}"
         loop_control:
           loop_var: firewall_object
-      when:
-        - awx_related is defined
-        - awx_related
 
     - name: "Setup hcloud firewalls for database backup stuff..."
       include_role:
@@ -91,6 +88,3 @@
       loop: "{{ hcloud_firewall_objects_backup }}"
       loop_control:
         loop_var: firewall_object
-      when:
-          - backup_related is defined
-          - backup_related

restore-remote-database-backup.yml

@@ -49,12 +49,12 @@
         - "stage_{{ stage }}"
         - 'restore'
       changed_when: False
-    - name: "Add 'storage' servers to hosts if necessary"
+    - name: "Add 'backup' servers to hosts if necessary"
       add_host:
-        name: "{{ stage }}-fgrz-01"
+        name: "{{ stage }}-backup-01"
         groups:
         - "stage_{{ stage }}"
-        - storage
+        - backup
       changed_when: False
 
 #############################################################
@@ -98,24 +98,61 @@
     - role: restore_{{ database_engine }}
 
 #############################################################
-#   Syncing backups from storage server to restore server
+#   add restore specific firewall rule
 #############################################################
 
-- hosts: "storage"
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  vars:
+    hcloud_firewall_objects_backup:
+       -
+         name: "{{ stage }}-restore-ssh-access"
+         state: present
+         rules:
+         -
+            direction: in
+            protocol: tcp
+            port: '22'
+            source_ips:
+              - "{{ lookup('community.general.dig', groups['backup'][0] + '.' + domain ) }}/32"
+            destination_ips: []
+            description: null
+         apply_to:
+         -
+           type: label_selector
+           label_selector:
+             selector: 'service=restore'
+
+  tasks:
+    - name: "Add hcloud firewall rule(s)"
+      include_role:
+        name: hcloud
+        tasks_from: configure-firewall2
+      loop: "{{ hcloud_firewall_objects_backup }}"
+      loop_control:
+        loop_var: firewall_object
+
+#############################################################
+#   Syncing backups from backup server to restore server
+#############################################################
+
+- hosts: "backup"
   serial: "{{ serial_number | default(5) }}"
   gather_facts: false
   vars:
-    storageserver_system_user: 'backuphamster'
+    backupserver_system_user: 'backuphamster'
     ansible_ssh_host: "{{ stage_server_domain }}"
   tasks:
     # I could not get it up and running with <synchronize> module
     # to sync data from remote server A to remote server B
     - name: "Syncing remote backups"
       become: yes
-      become_user: '{{ storageserver_system_user }}'
+      become_user: '{{ backupserver_system_user }}'
       vars:
-        database_server_ip: "{{ stage }}-restore-{{ database_engine }}-01.{{ domain }}"
+        database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}"
-      shell: '/home/{{ storageserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+      shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
 
 #############################################################
 #   Restoring from backup

roles/backup/files/pull_remote_backups.sh

@@ -11,4 +11,3 @@ DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/
 
 mkdir -p ${DEST_DIR}
 rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/
-

roles/backup/files/push_backups_to_restore_server.sh

@@ -12,14 +12,14 @@ DATABASE_ENGINE=$3
 DATE=$(date +%F)
 
 LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}"
-BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz | head -n 1)
+BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
 
 REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}"
 DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"
 
 SSH_OPTIONS='-o StrictHostKeyChecking=no'
 
-# needed due to unknown rsycn option --mkpath in rsycn version 3.1.3 
+# needed due to unknown rsync option --mkpath in rsync version 3.1.3
 ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}"
 
 rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR}

roles/backup/tasks/main.yml

@@ -29,8 +29,11 @@
 - name: "Providing rsync script"
   become: yes
   copy:
-    src: pull_remote_backups.sh
+    src: '{{ item }}'
-    dest: '/home/{{ system_user }}/pull_remote_backups.sh'
+    dest: '/home/{{ system_user }}/{{ item }}'
     mode: '0755'
     owner: '{{ system_user }}'
     group: '{{ system_user }}'
+  with_items:
+    - pull_remote_backups.sh
+    - push_backups_to_restore_server.sh

roles/management/tasks/configurations.yml

@@ -35,6 +35,7 @@
       path: "./smardigo/{{ item }}"
       dest: "{{ temp.path }}/{{ item }}.zip"
       format: zip
+      mode: '0644'
     with_items: "{{ connect_configurations }}"
     tags:
       - update_configurations

roles/restore_maria/files/restore.sh

@@ -15,7 +15,11 @@ systemctl stop mariadb
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}
 
-cat /home/backupuser/backups/${STAGE}/maria/${DATE}/mariabackupstream_${DATE}_*.gz | gunzip | mbstream --directory ${DATADIR} -x --parallel=2
+LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria"
+BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
+
+# --batch  => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" <<
+gpg --batch --decrypt $BACKUP_FILE_ENCRYPTED | gunzip | mbstream --directory ${DATADIR} -x --parallel=2
 
 mariabackup --prepare --target-dir=${DATADIR}
 

roles/restore_maria/tasks/main.yml

@@ -15,3 +15,20 @@
     mode: '0750'
     owner: root
     group: root
+
+- name: "Create file for gpg secret key"
+  become: yes
+  copy:
+    dest: '/root/gpg_private_key'
+    mode: '0600'
+    owner: 'root'
+    group: 'root'
+    content: |
+      {{ gpg_key_smardigo_automation__private }}
+
+- name: "Import private gpg key" # noqa command-instead-of-shell
+  become: yes
+  shell: 'gpg --import /root/gpg_private_key'
+  register: gpg_import
+  changed_when:
+    - gpg_import.rc != '0'

roles/restore_postgres/files/restore.sh

@@ -16,7 +16,11 @@ systemctl stop postgresql
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}
 
-tar -ixzf /home/backupuser/backups/${STAGE}/postgres/${DATE}/basebackup_${DATE}_*.tar.gz -C ${DATADIR}
+LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres"
+BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
+
+# --batch  => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" <<
+gpg --batch --decrypt $BACKUP_FILE_ENCRYPTED | tar -xz -C ${DATADIR}
 
 chmod 0700 ${DATADIR}
 chown -R ${PG_USER}:${PG_GROUP} ${DATADIR}

roles/restore_postgres/tasks/main.yml

@@ -10,3 +10,20 @@
     mode: 0754
     owner: root
     group: root
+
+- name: "Create file for gpg secret key"
+  become: yes
+  copy:
+    dest: '/root/gpg_private_key'
+    mode: '0600'
+    owner: 'root'
+    group: 'root'
+    content: |
+      {{ gpg_key_smardigo_automation__private }}
+
+- name: "Import private gpg key" # noqa command-instead-of-shell
+  become: yes
+  shell: 'gpg --import /root/gpg_private_key'
+  register: gpg_import
+  changed_when:
+    - gpg_import.rc != '0'

smardigo.yml

@@ -71,3 +71,6 @@
 
     - role: pdns
       when: "'pdns' in group_names"
+
+    - role: backup
+      when: "'backup' in group_names"

stage-dev

@@ -1,5 +1,6 @@
-[bastelserver]
+[backup]
 dev-fgrz-01
+dev-backup-01
 
 [connect]
 dev-management-01
@@ -76,7 +77,7 @@ kube_control_plane
 kube_node
 
 [stage_dev:children]
-bastelserver
+backup
 connect
 elastic
 pdns

stage-prodnso

@@ -1,3 +1,6 @@
+[backup]
+prodnso-backup-01
+
 [connect]
 prodnso-management-01
 
@@ -72,6 +75,7 @@ kube_control_plane
 kube_node
 
 [stage_prodnso:children]
+backup
 connect
 elastic
 pdns

stage-qa

@@ -1,3 +1,6 @@
+[backup]
+qa-backup-01
+
 [connect]
 qa-management-01
 
@@ -72,6 +75,7 @@ kube_control_plane
 kube_node
 
 [stage_qa:children]
+backup
 connect
 elastic
 pdns

storage_server.yml

@@ -1,4 +0,0 @@
----
-- hosts: bastelserver
-  roles:
-    - role: storage_server

users/backuphamster/ssh.pub

@@ -0,0 +1 @@
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFRlmqgkIJxBC45cbVX25P1Uam/+Ct7XFvgMm60TDOWkQiTuVp5vd1sHq2HCRRfGxPrsKmwSQS5wMYIjeiclTag= friedrich@friedrich-HP-ZBook