DEV-371: added stuff to do remote backup

create-remote-database-backup.yml

@@ -0,0 +1,113 @@
+---
+
+# creates remote database backup
+#   - postgres
+#     - executed on stage specific server: {{ stage }}-postgres-02 (currently: slave)
+#     - creates database backup for ALL databases in postgres-server
+#   - mariadb
+#     - executed on stage specific server: {{ stage }}-maria-01
+#     - creates database backup for ALL databases in mariadb-server
+
+# Parameters:
+#   playbook inventory
+#     stage := the name of the stage (e.g. dev, int, qa, prod)
+#   smardigo message callback
+#     scope_id := (scope id of the management process)
+#     process_instance_id := (process instance id of the management process)
+#     smardigo_management_action := (smardigo management action anme of the management process)
+
+#############################################################
+#   Creating inventory dynamically for given parameters
+#############################################################
+
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+
+#     add virtual server to load stage specific variables as context
+    - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
+      add_host:
+        name: "{{ stage }}-virtual-host-to-read-groups-vars"
+        groups:
+        - "stage_{{ stage }}"
+      changed_when: False
+
+  tasks:
+    - name: "Add {{ database_engine }} servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-{{ database_engine }}-01"
+        groups:
+        - "stage_{{ stage }}"
+        - '{{ database_engine }}'
+      changed_when: False
+    - name: "Add 'storage' servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-fgrz-01"
+        groups:
+        - "stage_{{ stage }}"
+        - storage
+      changed_when: False
+
+##############################################################
+##   Creating remote database backups for created inventory
+##############################################################
+
+- hosts: "postgres:maria"
+  serial: "{{ serial_number | default(1) }}"
+  tasks:
+    - name: "Trigger backup mechanism"
+      include_role:
+        name: '{{ database_engine }}'
+        tasks_from: _create_backup
+
+#############################################################
+#   Syncing remote database backups to storage server
+#############################################################
+
+- hosts: "postgres:maria:storage"
+  serial: "{{ serial_number | default(5) }}"
+  vars:
+    storageserver_system_user: 'backuphamster'
+  tasks:
+    # I could not get it up and running with <synchronize> module
+    # to sync data from remote server A to remote server B
+    - name: "Syncing remote backups"
+      become: yes
+      become_user: '{{ storageserver_system_user }}'
+      vars:
+        database_server_ip: "{{ stage }}-{{ database_engine }}-01.{{ domain }}"
+      shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+      when:
+        - inventory_hostname in groups['storage']
+
+    - name: "Cleanup remote backup dirs: {{ database_engine }}"
+      become: yes
+      file:
+        path: '{{ backup_directory }}/{{ database_engine }}/{{ ansible_date_time.date }}'
+        state: absent
+      when:
+        - not inventory_hostname in groups['storage']
+
+#############################################################
+#   Sending smardigo management message to process
+#############################################################
+
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  run_once: true
+  vars:
+    connect_jwt_username: "{{ management_admin_username }}"
+
+  tasks:
+    - name: "Sending smardigo management message to <{{ smardigo_management_url }}>"
+      include_tasks: tasks/smardigo_management_message.yml

group_vars/all/plain.yml

@@ -99,6 +99,7 @@ default_plattform_users:
   - 'postgres'
   - 'administrator'
   - '{{ admin_user }}'
+  - '{{ backupuser_username }}'
 
 smardigo_plattform_users:
   - 'gitlabci' # needed for periodic ansible run
@@ -117,6 +118,10 @@ ip_whitelist:
   - "46.245.219.98/32" # netgo borken
   - "{{ shared_service_network }}"
 
+# for test purpose DEV-361
+# currently (march2022) set to IP of hetzner VM
+gitlab_storage_server: 167.235.18.147/32
+
 docker_owner: "{{ admin_user }}"
 docker_group: "{{ admin_user }}"
 docker_users: "{{ smardigo_plattform_users }}"
@@ -185,6 +190,10 @@ blackbox_http_2xx_additional_targets: []
 prometheus_federation_enabled: true
 kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
 
+backupuser_username: backupuser
+backupuser_ssh_pubkey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDAFRYAy3PqimYUWcO4Q9pdTvDQTsq7hKjWYoQEsJICnRRv+W+5d2lJvC3gqMpmWy9XxtrYePkVHCgIvfJSas9Jv7n7eeYoeWLWJq0nRSKg6EKFCH9y3v8tGPJQQf7wogOhHwr6m79c+lpNVUsVR+QOf76+47ZuwnuEBzK6xbDkmwyt7SPrJ59IFxOlmtz2HgVlTLczLalMygM4qlXqIt+lwuuFz4CsGcr4TwMKp9Uk6SCP3OV12oLnUUUOA3r72qmE4+JeUN6VNbXoBXEANfXm5kbM8w+dFhulCi1fQZCssB8PStA7Cs0gVqL6DYNUKRZaFL8e77hljGkPlOQDxOsBexPuceSDmmr6s5qT1wA6bnEFoeWbLlxixGlFA+1Q/LqWsYzoOZiTHDoaXvsc4VizlPp4Fn0OgJefPjuzBsWOyf0ob5oucfnmCAvEh/k+ioq0bIQDcliAM1UezitblHQgGHhqnKPMi664i0ULLiExARe4IV3KJiaG++RJyzUL5HNz3Qru+K5/pdj2jffluYTC4w+6ZYfjWEZS/DAumExv9T97kFOsapHCQJwTBa368Ch6uKkPCZO8p/ra3xTIUh/PibHaVCadgX2NR9q6jdiQtmc0SOyNJlMlPZD/Q1NrjXJ18ASny7gCBFItMyMtinVx9xQxQ+PFLB8oNYERw1ejIw== storage-server-smardigo'
+
+current_date_time: "{{ lookup('pipe','date +%Y-%m-%d_%H:%M') }}"
 
 hcloud_firewall_objects:
    -
@@ -362,3 +371,26 @@ hcloud_firewall_objects_awx:
        type: label_selector
        label_selector:
          selector: 'service=kibana'
+
+hcloud_firewall_objects_backup:
+   -
+     name: "{{ stage }}-database-backup-ssh-access"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '22'
+        source_ips:
+          - "{{ gitlab_storage_server }}"
+        destination_ips: []
+        description: null
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'service=postgres'
+     -
+       type: label_selector
+       label_selector:
+         selector: 'service=maria'

group_vars/bastelserver/plain.yml

@@ -0,0 +1,9 @@
+---
+#TODO needs to be removed after story DEV-361 is finished
+hetzner_server_type: "{{ hetzner_server_type_bastelserver | default('cx21') }}"
+hetzner_server_labels: "stage={{ stage }} service=bastelserver"
+
+docker_enabled: false
+traefik_enabled: false
+filebeat_enabled: false
+node_exporter_enabled: false

hcloud_firewall.yml

@@ -81,4 +81,16 @@
         loop_control:
           loop_var: firewall_object
       when:
+        - awx_related is defined
         - awx_related
+
+    - name: "Setup hcloud firewalls for database backup stuff..."
+      include_role:
+        name: hcloud
+        tasks_from: configure-firewall2
+      loop: "{{ hcloud_firewall_objects_backup }}"
+      loop_control:
+        loop_var: firewall_object
+      when:
+          - backup_related is defined
+          - backup_related

roles/common/tasks/main.yml

@@ -97,6 +97,23 @@
   tags:
     - users
 
+- name: "Create stuff for backups on database servers"
+  block:
+  - name: "Create system user for remote_backup"
+    become: yes
+    ansible.builtin.user:
+      name: '{{ backupuser_username }}'
+      comment: "user for backup"
+      shell: /bin/bash
+
+  - name: "Add SSH pub key to auth_keys"
+    authorized_key:
+      user: '{{ backupuser_username }}'
+      key: '{{ backupuser_ssh_pubkey }}'
+  when:
+    - inventory_hostname in groups['postgres'] or
+      inventory_hostname in groups['maria']
+
 - name: "Ensure docker configuration directory exists"
   file:
     path: '/home/{{ item }}/.docker/'

roles/connect_postgres/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

roles/gitea_postgres/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

roles/keycloak_postgres/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

roles/maria/defaults/main.yml

@@ -1 +1,5 @@
 ---
+my_cnf_file: '/root/.my.cnf'
+database_engine: maria
+backup_dest_dir: "{{ backup_directory }}/{{ database_engine }}/{{ ansible_date_time.date }}"
+backup_status_file: '{{ backup_dest_dir }}/backup_finished'

roles/maria/tasks/_create_backup.yml

@@ -0,0 +1,57 @@
+---
+- name: "Create destination backup directory"
+  become: yes
+  ansible.builtin.file:
+    path: '{{ backup_dest_dir }}'
+    state: directory
+    mode: '0755'
+    owner: root
+    group: root
+
+- name: "Create {{ my_cnf_file }} file"
+  become: yes
+  copy:
+    dest: '{{ my_cnf_file }}'
+    mode: '0600'
+    content: |
+      [client]
+      user={{ mysql_root_username }}
+      password={{ mysql_root_password }}
+
+# there is no ansible module already in place
+# so using shell module
+- name: "Creating mariabackup ... + doing async check if successful or not"
+  become: yes
+  shell: |
+      set -o pipefail
+      /usr/bin/mariabackup --defaults-file={{ my_cnf_file }} --backup --stream=xbstream | gzip > {{ backup_dest_dir }}/mariabackupstream_{{ current_date_time }}.gz
+  args:
+    executable: /bin/bash
+  async: 3600 # allows duration for task up to 3600sec
+  poll: 30 # rechecks every 30sec if task has finished yet
+  changed_when: false
+
+# just to make it easier to detect potential failures.
+# maybe: can be removed later
+- name: "Create STATUS file for successful backup"
+  become: yes
+  file:
+    path: '{{ backup_status_file }}_{{ current_date_time }}'
+    state: touch
+    mode: '0644'
+    owner: root
+    group: root
+
+- name: "Prepare backup dir..."
+  become: yes
+  ansible.builtin.file:
+    path: '{{ backup_dest_dir }}'
+    owner: '{{ backupuser_username }}'
+    group: '{{ backupuser_username }}'
+    recurse: yes
+
+- name: "Remove {{ my_cnf_file }} file"
+  become: yes
+  file:
+    path: '{{ my_cnf_file }}'
+    state: absent

roles/maria/tasks/main.yml

@@ -97,3 +97,9 @@
     name: prometheus-mysqld-exporter
     state: started
     enabled: yes
+
+- name: 'Ensures <{{ backup_directory }}> directory exists'
+  file:
+    state: directory
+    path: '{{ backup_directory }}'
+    mode: 0755

roles/postgres/defaults/main.yml

@@ -17,3 +17,7 @@ default_shared_buffers: 256MB
 database_state: present
 
 postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_private_server_ip }}'"
+
+database_engine: postgres
+backup_dest_dir: "{{ backup_directory }}/{{ database_engine }}/{{ ansible_date_time.date }}"
+backup_status_file: '{{ backup_dest_dir }}/backup_finished'

roles/postgres/tasks/_create_backup.yml

@@ -0,0 +1,47 @@
+---
+- name: "Ensure needed packages"
+  become: yes
+  package:
+    name: pigz
+
+- name: "Create destination backup directory"
+  become: yes
+  ansible.builtin.file:
+    path: '{{ backup_dest_dir }}'
+    state: directory
+    mode: '0755'
+    owner: postgres
+    group: postgres
+
+# there is no ansible module already in place
+# so using shell module
+- name: "Creating pg_basebackup ... + doing async check if successful or not"
+  become: yes
+  become_user: postgres
+  shell: |
+      set -o pipefail
+      /usr/bin/pg_basebackup -Ft -X fetch -D - | pigz -p 4 > {{ backup_dest_dir }}/basebackup_{{ current_date_time }}.tar.gz
+  args:
+    executable: /bin/bash
+  async: 3600 # allows duration for task up to 3600sec
+  poll: 30 # rechecks every 30sec if task has finished yet
+  changed_when: false
+
+# just to make it easier to detect potential failures.
+# maybe: can be removed later
+- name: "Create STATUS file for successful backup"
+  become: yes
+  file:
+    path: '{{ backup_status_file }}_{{ current_date_time }}'
+    state: touch
+    mode: '0644'
+    owner: postgres
+    group: postgres
+
+- name: "Prepare backup dir..."
+  become: yes
+  ansible.builtin.file:
+    path: '{{ backup_dest_dir }}'
+    owner: '{{ backupuser_username }}'
+    group: '{{ backupuser_username }}'
+    recurse: yes

roles/storage_server/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+system_user: backuphamster

roles/storage_server/files/pull_remote_backups.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+#
+#
+
+REMOTE_SYSTEM_USER=backupuser
+DATABASE_SERVER_IP=$1
+STAGE=$2
+DATABASE_ENGINE=$3
+DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/
+
+mkdir -p ${DEST_DIR}
+rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/* ${DEST_DIR}/
+

roles/storage_server/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+- name: "Backup storage server | create system user"
+  become: yes
+  ansible.builtin.user:
+    name: '{{ system_user }}'
+    comment: "user for backup"
+    shell: /bin/bash
+  register: create_user
+
+- name: "Create .ssh dir"
+  become: yes
+  file:
+    path: '/home/{{ system_user }}/.ssh/'
+    mode: '0700'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    state: directory
+
+- name: "Providing SSH priv.key"
+  no_log: true
+  become: yes
+  copy:
+    dest: '/home/{{ system_user }}/.ssh/id_rsa'
+    mode: '0400'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    content: '{{ backup_user_ssh_privkey_vault }}'
+
+- name: "Providing rsync script"
+  become: yes
+  copy:
+    src: pull_remote_backups.sh
+    dest: '/home/{{ system_user }}/pull_remote_backups.sh'
+    mode: '0755'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'

roles/webdav_postgres/tasks/main.yml

@@ -12,7 +12,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

roles/workflow_index_postgres/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

roles/workflow_proxy_postgres/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _postgres-backups.yml
+    tasks_from: _create_database_backup.yml
   when:
   - postgres_backup_state is defined
   - postgres_backup_state in ['dump', 'restore']

stage-dev

@@ -1,3 +1,6 @@
+[bastelserver]
+dev-fgrz-01
+
 [connect]
 dev-management-01
 
@@ -73,6 +76,7 @@ kube_control_plane
 kube_node
 
 [stage_dev:children]
+bastelserver
 connect
 elastic
 pdns

storage_server.yml

@@ -0,0 +1,4 @@
+---
+- hosts: bastelserver
+  roles:
+    - role: storage_server