DEV-373: try to automate restore from database backup

create-server.yml

@@ -52,6 +52,7 @@
 
 - hosts: "stage_{{ stage }}:!{{ stage }}-virtual-host-to-read-groups-vars"
   serial: "{{ serial_number | default(5) }}"
+  remote_user: root
   gather_facts: false
 
   pre_tasks:

galaxy-requirements.yml

@@ -20,6 +20,8 @@ roles:
   version: v3.6.1
   src: https://github.com/Oefenweb/ansible-postfix.git
   scm: git
+- name: geerlingguy.mysql
+  version: 3.3.2
 
 collections:
 - name: hetzner.hcloud

group_vars/all/plain.yml

@@ -402,3 +402,7 @@ hcloud_firewall_objects_backup:
        type: label_selector
        label_selector:
          selector: 'service=maria'
+     -
+       type: label_selector
+       label_selector:
+         selector: 'service=restore'

group_vars/restore/plain.yml

@@ -0,0 +1,16 @@
+---
+
+hetzner_server_type: "{{ hetzner_server_type_restore_database | default('cpx21') }}"
+hetzner_server_labels: "stage={{ stage }} service=restore database_engine={{ database_engine | default('') }}"
+
+docker_enabled: false
+traefik_enabled: false
+filebeat_enabled: false
+node_exporter_enabled: true
+
+custom_plattform_users:
+  - '{{ backupuser_user_name }}'
+
+# postgresql related
+# defining type of server (naster|slave|restore)
+server_type: restore

group_vars/stage_dev/plain.yml

@@ -366,7 +366,7 @@ management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
 # smardigo automation DEV gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://dev-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
 
 iam_opentracing_jaeger_enabled: true

restore-remote-database-backup.yml

@@ -0,0 +1,150 @@
+---
+
+# restores remote database backup
+#   - postgres
+#     - executed on stage specific server: {{ stage }}-restore-postgres-01
+#     - restores a server from full-backup
+#   - mariadb
+#     - executed on stage specific server: {{ stage }}-restore-maria-01
+#     - restores a server from full-backup
+
+# Parameters:
+#   playbook inventory
+#     stage := the name of the stage (e.g. dev, int, qa, prod)
+#     database_engine := the database engine to restore a backup for (e.g. postgres, maria)
+#   smardigo message callback
+#     scope_id := (scope id of the management process)
+#     process_instance_id := (process instance id of the management process)
+#     smardigo_management_action := (smardigo management action anme of the management process)
+
+#############################################################
+#   Creating inventory dynamically for given parameters
+#############################################################
+
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+
+#     add virtual server to load stage specific variables as context
+    - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
+      add_host:
+        name: "{{ stage }}-virtual-host-to-read-groups-vars"
+        groups:
+        - "stage_{{ stage }}"
+      changed_when: False
+
+  tasks:
+    - name: "Add {{ database_engine }} servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-restore-{{ database_engine }}-01"
+        groups:
+        - "stage_{{ stage }}"
+        - 'restore'
+      changed_when: False
+    - name: "Add 'storage' servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-fgrz-01"
+        groups:
+        - "stage_{{ stage }}"
+        - storage
+      changed_when: False
+
+#############################################################
+#   Create restore server(s)
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  remote_user: root
+
+  roles:
+    - role: hcloud
+
+#############################################################
+#   Provisioning server(s) for created inventory
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  remote_user: root
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+
+  pre_tasks:
+    - name: "Import autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      become: false
+      tags:
+        - always
+
+  roles:
+    - role: common
+
+    - role: filebeat
+      when: filebeat_enabled | default(True)
+
+    - role: node_exporter
+      when: node_exporter_enabled | default(True)
+
+    - role: restore_{{ database_engine }}
+
+#############################################################
+#   Syncing backups from storage server to restore server
+#############################################################
+
+- hosts: "storage"
+  serial: "{{ serial_number | default(5) }}"
+  gather_facts: false
+  vars:
+    storageserver_system_user: 'backuphamster'
+    ansible_ssh_host: "{{ stage_server_domain }}"
+  tasks:
+    # I could not get it up and running with <synchronize> module
+    # to sync data from remote server A to remote server B
+    - name: "Syncing remote backups"
+      become: yes
+      become_user: '{{ storageserver_system_user }}'
+      vars:
+        database_server_ip: "{{ stage }}-restore-{{ database_engine }}-01.{{ domain }}"
+      shell: '/home/{{ storageserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+
+#############################################################
+#   Restoring from backup
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+  tasks:
+    - name: "Triggering restore"
+      become: yes
+      shell: '/root/restore.sh {{ stage }}'
+
+# TODO delete restore-server
+
+#############################################################
+#   Sending smardigo management message to process
+#############################################################
+
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  run_once: true
+  vars:
+    connect_jwt_username: "{{ management_admin_username }}"
+
+  tasks:
+    - name: "Sending smardigo management message to <{{ smardigo_management_url }}>"
+      include_tasks: tasks/smardigo_management_message.yml

roles/common/tasks/main.yml

@@ -97,6 +97,14 @@
   tags:
     - users
 
+- name: "Update available package list"
+  apt:
+    update_cache: yes
+  tags:
+    - install
+    - upgrade
+  when: ansible_distribution == "Ubuntu"
+
 - name: "Ensure docker configuration directory exists"
   file:
     path: '/home/{{ item }}/.docker/'

roles/hcloud/tasks/_set_server_state.yml

@@ -12,3 +12,42 @@
     state: "{{ server_state }}"
   delegate_to: 127.0.0.1
   become: false
+  async: 300
+  poll: 5
+  register: hcloud_response
+  ignore_errors: yes
+
+# poweron server if server not running
+- name: "Ensure Server is STARTED when server_state=present"
+  hetzner.hcloud.hcloud_server:
+    api_token: "{{ hetzner_authentication_ansible }}"
+    name: "{{ inventory_hostname }}"
+    state: "started"
+  delegate_to: 127.0.0.1
+  become: false
+  async: 300
+  poll: 15
+  when:
+    - server_state == 'present'
+
+# just catching hetzner error and trying to avoid manual interaction
+- name: "DEBUGGING Block for hetzner << message not found >> - error"
+  block:
+  - name: "DEBUG: hcloud_response"
+    debug:
+      msg: hcloud_response.msg
+
+  - name: "Ensure Server is STARTED when server_state=present"
+    hetzner.hcloud.hcloud_server:
+      api_token: "{{ hetzner_authentication_ansible }}"
+      name: "{{ inventory_hostname }}"
+      state: "started"
+    delegate_to: 127.0.0.1
+    become: false
+    async: 300
+    poll: 15
+    when:
+      - server_state == 'present'
+
+  when:
+    - hcloud_response.msg is defined

roles/maria/tasks/_create_backup.yml

@@ -18,13 +18,17 @@
       user={{ mysql_root_username }}
       password={{ mysql_root_password }}
 
-# there is no ansible module already in place
+# there is no ansible module already in place for (mariabackup|gpg)
 # so using shell module
 - name: "Creating mariabackup ... + doing async check if successful or not"
   become: yes
+  vars:
+    backup_file: '{{ backup_dest_dir }}/mariabackupstream_{{ current_date_time }}.gz'
   shell: |
       set -o pipefail
-      /usr/bin/mariabackup --defaults-file={{ my_cnf_file }} --backup --stream=xbstream | gzip > {{ backup_dest_dir }}/mariabackupstream_{{ current_date_time }}.gz
+      /usr/bin/mariabackup --defaults-file={{ my_cnf_file }} --backup --stream=xbstream | gzip > {{ backup_file }} && \
+      gpg --encrypt --recipient "smardigo automation {{ stage | upper }}" --trust-model always {{ backup_file }} && \
+      rm {{ backup_file }}
   args:
     executable: /bin/bash
   async: 3600 # allows duration for task up to 3600sec

roles/maria/tasks/main.yml

@@ -103,3 +103,23 @@
     state: directory
     path: '{{ backup_directory }}'
     mode: 0755
+
+- name: "Block: gpg stuff"
+  block:
+    - name: Create temp dir
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: gitcheckout
+        path: /tmp
+      register: tempdir
+    
+    - name: "Checkout repo for gpg communication-keys"
+      ansible.builtin.git:
+        repo: 'https://{{ gituser | default("gitea-admin") | urlencode }}:{{ gitea_admin_password | urlencode }}@{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys.git'
+        dest: '{{ tempdir.path }}'
+        version: master
+    
+# there is no ansible gpg module already in place
+# linting violation needs to be whitelisted
+    - name: "Importing stage specific automation gpg-key" # noqa command-instead-of-shell
+      shell: 'gpg --import {{ tempdir.path }}/smardigo_automation_{{ stage }}.gpg.pub'

roles/postgres/tasks/_create_backup.yml

@@ -13,14 +13,18 @@
     owner: postgres
     group: postgres
 
-# there is no ansible module already in place
+# there is no ansible module already in place for (pg_basebackup|gpg)
 # so using shell module
 - name: "Creating pg_basebackup ... + doing async check if successful or not"
   become: yes
   become_user: postgres
+  vars:
+    backup_file: '{{ backup_dest_dir }}/basebackup_{{ current_date_time }}.tar.gz'
   shell: |
       set -o pipefail
-      /usr/bin/pg_basebackup -Ft -X fetch -D - | pigz -p 4 > {{ backup_dest_dir }}/basebackup_{{ current_date_time }}.tar.gz
+      /usr/bin/pg_basebackup -Ft -X fetch -D - | pigz -p 4 > {{ backup_file }} && \
+      gpg --encrypt --recipient "smardigo automation {{ stage | upper }}" --trust-model always {{ backup_file }} && \
+      rm {{ backup_file }}
   args:
     executable: /bin/bash
   async: 3600 # allows duration for task up to 3600sec

roles/postgres/tasks/base-requirements.yml

@@ -149,3 +149,25 @@
   service:
     name: prometheus-postgres-exporter
     state: restarted
+
+- name: "Block: gpg stuff"
+  become: yes
+  become_user: postgres
+  block:
+    - name: Create temp dir
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: gitcheckout
+        path: /tmp
+      register: tempdir
+
+    - name: "Checkout repo for gpg communication-keys"
+      ansible.builtin.git:
+        repo: 'https://{{ gituser | default("gitea-admin") | urlencode }}:{{ gitea_admin_password | urlencode }}@{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys.git'
+        dest: '{{ tempdir.path }}'
+        version: master
+
+# there is no ansible gpg module already in place
+# linting violation needs to be whitelisted
+    - name: "Importing stage specific automation gpg-key" # noqa command-instead-of-shell
+      shell: 'gpg --import {{ tempdir.path }}/smardigo_automation_{{ stage }}.gpg.pub'

roles/restore_maria/files/restore.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+#
+#
+#
+
+STAGE=$1
+
+DATADIR='/var/lib/mysql'
+DATE=$(date +%F)
+
+
+systemctl stop mariadb
+
+mv ${DATADIR} ${DATADIR}_moved
+mkdir -p ${DATADIR}
+
+cat /home/backupuser/backups/${STAGE}/maria/${DATE}/mariabackupstream_${DATE}_*.gz | gunzip | mbstream --directory ${DATADIR} -x --parallel=2
+
+mariabackup --prepare --target-dir=${DATADIR}
+
+chown -R mysql:mysql ${DATADIR}
+
+systemctl start mariadb

roles/restore_maria/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: "Install mariadb via include_role"
+  vars:
+    mysql_packages:
+      - mariadb-client
+      - mariadb-server
+      - mariadb-backup
+  include_role:
+    name: geerlingguy.mysql
+
+- name: "Copy restore script to restore server"
+  copy:
+    src: restore.sh
+    dest: '/root/restore.sh'
+    mode: '0750'
+    owner: root
+    group: root

roles/restore_postgres/files/restore.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+#
+#
+
+STAGE=$1
+
+DATADIR='/var/lib/postgresql/13/main'
+DATE=$(date +%F)
+
+PG_USER=postgres
+PG_GROUP=postgres
+
+systemctl stop postgresql
+
+mv ${DATADIR} ${DATADIR}_moved
+mkdir -p ${DATADIR}
+
+tar -ixzf /home/backupuser/backups/${STAGE}/postgres/${DATE}/basebackup_${DATE}_*.tar.gz -C ${DATADIR}
+
+chmod 0700 ${DATADIR}
+chown -R ${PG_USER}:${PG_GROUP} ${DATADIR}
+
+systemctl start postgresql

roles/restore_postgres/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: "Install postgres via include_role"
+  include_role:
+    name: postgres
+
+- name: "Copy restore script to restore server"
+  copy:
+    src: restore.sh
+    dest: '/root/restore.sh'
+    mode: 0754
+    owner: root
+    group: root

roles/storage_server/files/push_backups_to_restore_server.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+#
+#
+
+REMOTE_SYSTEM_USER=backupuser
+DATABASE_SERVER_IP=$1
+STAGE=$2
+DATABASE_ENGINE=$3
+
+# currently it defaults to todays date
+DATE=$(date +%F)
+
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}"
+BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz | head -n 1)
+
+REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}"
+DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"
+
+SSH_OPTIONS='-o StrictHostKeyChecking=no'
+
+# needed due to unknown rsycn option --mkpath in rsycn version 3.1.3 
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}"
+
+rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR}
+
+BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}')
+
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}"