--- # restores remote database backup # - postgres # - executed on stage specific server: {{ shared_service_postgres_primary }}-restore # - restores a server from full-backup # - mariadb # - executed on stage specific server: {{ shared_service_maria_primary }}-restore # - restores a server from full-backup # Parameters: # playbook inventory # stage := the name of the stage (e.g. dev, int, qa, prod) # database_engine := the database engine to restore a backup for (e.g. postgres, maria) # smardigo message callback # scope_id := (scope id of the management process) # process_instance_id := (process instance id of the management process) # smardigo_management_action := (smardigo management action anme of the management process) ############################################################# # Creating inventory dynamically for given parameters ############################################################# - hosts: localhost connection: local gather_facts: false pre_tasks: - name: "Import constraints check" import_tasks: tasks/constraints_check.yml tags: - always # add virtual server to load stage specific variables as context - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts" add_host: name: "{{ stage }}-virtual-host-to-read-groups-vars" groups: - "stage_{{ stage }}" changed_when: False tasks: - name: "Add {{ database_engine }}-restore servers to hosts if necessary" add_host: name: "{{ shared_service_postgres_secondary }}-restore" groups: - "stage_{{ stage }}" - 'restore' when: - database_engine in ['postgres'] - name: "Add {{ database_engine }}-restore servers to hosts if necessary" add_host: name: "{{ shared_service_maria_primary }}-restore" groups: - "stage_{{ stage }}" - 'restore' when: - database_engine in ['maria'] - name: "Add 'backup' servers to hosts if necessary" add_host: name: "{{ stage }}-backup-01" groups: - "stage_{{ stage }}" - backup changed_when: False ############################################################# # Create restore server(s) ############################################################# - hosts: "restore" serial: "{{ serial_number | default(1) }}" gather_facts: false remote_user: root roles: - role: hetzner-ansible-hcloud vars: sma_digitalocean_ttl: 60 # set it to 60sec to reduce DNS caching problems with internal IT in case of debugging ansible problems ;) - role: hetzner-ansible-dns vars: record_data: "{{ stage_server_ip }}" record_name: "{{ inventory_hostname }}" tags: - update_dns ############################################################# # Provisioning server(s) for created inventory ############################################################# - hosts: "restore" serial: "{{ serial_number | default(1) }}" remote_user: root vars: ansible_ssh_host: "{{ stage_server_domain }}" pre_tasks: - name: "Import autodiscover pre-tasks" import_tasks: tasks/autodiscover_pre_tasks.yml become: false tags: - always roles: - role: hetzner-ansible-common - role: devsec.hardening.ssh_hardening tags: - ssh_hardening - role: hetzner-ansible-filebeat when: filebeat_enabled | default(True) - role: hetzner-ansible-node-exporter when: node_exporter_enabled | default(True) - role: restore_{{ database_engine }} ############################################################# # add restore specific firewall rule ############################################################# - hosts: "{{ stage }}-virtual-host-to-read-groups-vars" serial: "{{ serial_number | default(1) }}" gather_facts: false connection: local vars: hcloud_firewall_objects_backup: - name: "{{ stage }}-restore-ssh-access" state: present rules: - direction: in protocol: tcp port: '22' source_ips: - "{{ lookup('community.general.dig', groups['backup'][0] + '.' + domain ) }}/32" destination_ips: [] description: null apply_to: - type: label_selector label_selector: selector: 'service=restore' tasks: - name: "Add hcloud firewall rule(s)" include_role: name: hetzner-ansible-hcloud tasks_from: configure-firewall2 loop: "{{ hcloud_firewall_objects_backup }}" loop_control: loop_var: firewall_object ############################################################# # Syncing backups from backup server to restore server ############################################################# - hosts: "backup" serial: "{{ serial_number | default(5) }}" gather_facts: false vars: backupserver_system_user: 'backuphamster' ansible_ssh_host: "{{ stage_server_domain }}" tasks: # I could not get it up and running with module # to sync data from remote server A to remote server B - name: "Syncing remote backups" become: yes become_user: '{{ backupserver_system_user }}' vars: database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}" shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}' ############################################################# # Restoring from backup ############################################################# - hosts: "restore" serial: "{{ serial_number | default(1) }}" gather_facts: false vars: ansible_ssh_host: "{{ stage_server_domain }}" tasks: - name: "Triggering restore" become: yes shell: '/root/restore.sh {{ stage }}' - name: "Check for test data on postgres" block: - name: "Querying postgres ..." become: yes become_user: postgres community.postgresql.postgresql_query: db: dummytestdb query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s named_args: quote_val: 'Shall we play' register: query_output - assert: that: - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1' when: - database_engine == 'postgres' - name: "Check for test data on mariadb" block: - name: "Querying mariadb ..." become: yes become_user: root community.mysql.mysql_query: login_unix_socket: /run/mysqld/mysqld.sock login_db: dummytestdb query: SELECT movie FROM movie_quotes WHERE quote = %s positional_args: - 'Shall we play' register: query_output - assert: that: - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1' when: - database_engine == 'maria' ############################################################# # Deleting volumes for created inventory ############################################################# - hosts: "restore" serial: "{{ serial_number | default(5) }}" gather_facts: false tasks: - name: "Delete volumes for <{{ inventory_hostname }}>" include_role: name: lvm_with_hetzner_volumes tasks_from: _remove_hetzner_volumes ############################################################# # Deleting servers/domains for created inventory ############################################################# - hosts: "restore" serial: "{{ serial_number | default(5) }}" gather_facts: false tasks: - name: "Delete server <{{ inventory_hostname }}>" include_role: name: hetzner-ansible-hcloud tasks_from: _set_server_state vars: - server_state: "absent" - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>" include_role: name: hetzner-ansible-dns tasks_from: _remove_dns vars: record_to_remove: '{{ inventory_hostname }}' ############################################################# # Sending metric to prometheus push gateway ############################################################# - hosts: "{{ stage }}-virtual-host-to-read-groups-vars" serial: "{{ serial_number | default(1) }}" gather_facts: false connection: local run_once: true vars: prometheus_pushgw_host: prometheus-pushgateway.monitoring:9091 # pushgw-endpoint within k8s cluster running AWX tasks: # attention: do not remove manually set newline char at the end of metric line # in case of removing: request will die with HTTP 400 bad request - name: "Set Fact" set_fact: metric: "nightly_restore_successful_generic {{ lookup('pipe', 'date +%s') }}\n" # body => trim needed to remove new line char # due to created yaml string - name: "Send metric to prometheus pushgw" uri: url: "http://{{ prometheus_pushgw_host }}/metrics/job/restore_test/creator/awx-run/database_engine/{{ database_engine }}" method: POST body: "{{ metric }}" status_code: [200] register: send_prome_metric retries: 5 delay: 5 until: send_prome_metric.status in [200] ############################################################# # Sending smardigo management message to process ############################################################# - hosts: "{{ stage }}-virtual-host-to-read-groups-vars" serial: "{{ serial_number | default(1) }}" gather_facts: false connection: local run_once: true vars: connect_jwt_username: "{{ management_admin_username }}" tasks: - name: "Sending smardigo management message to <{{ shared_service_url_management }}>" include_tasks: tasks/smardigo_management_message.yml