You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
317 lines
9.8 KiB
YAML
317 lines
9.8 KiB
YAML
---
|
|
|
|
# restores remote database backup
|
|
# - postgres
|
|
# - executed on stage specific server: {{ shared_service_postgres_primary }}-restore
|
|
# - restores a server from full-backup
|
|
# - mariadb
|
|
# - executed on stage specific server: {{ shared_service_maria_primary }}-restore
|
|
# - restores a server from full-backup
|
|
|
|
# Parameters:
|
|
# playbook inventory
|
|
# stage := the name of the stage (e.g. dev, int, qa, prod)
|
|
# database_engine := the database engine to restore a backup for (e.g. postgres, maria)
|
|
# smardigo message callback
|
|
# scope_id := (scope id of the management process)
|
|
# process_instance_id := (process instance id of the management process)
|
|
# smardigo_management_action := (smardigo management action anme of the management process)
|
|
|
|
#############################################################
|
|
# Creating inventory dynamically for given parameters
|
|
#############################################################
|
|
|
|
- hosts: localhost
|
|
connection: local
|
|
gather_facts: false
|
|
|
|
pre_tasks:
|
|
- name: "Import constraints check"
|
|
import_tasks: tasks/constraints_check.yml
|
|
tags:
|
|
- always
|
|
|
|
# add virtual server to load stage specific variables as context
|
|
- name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
|
|
add_host:
|
|
name: "{{ stage }}-virtual-host-to-read-groups-vars"
|
|
groups:
|
|
- "stage_{{ stage }}"
|
|
changed_when: False
|
|
|
|
tasks:
|
|
- name: "Add {{ database_engine }}-restore servers to hosts if necessary"
|
|
add_host:
|
|
name: "{{ shared_service_postgres_secondary }}-restore"
|
|
groups:
|
|
- "stage_{{ stage }}"
|
|
- 'restore'
|
|
when:
|
|
- database_engine in ['postgres']
|
|
|
|
- name: "Add {{ database_engine }}-restore servers to hosts if necessary"
|
|
add_host:
|
|
name: "{{ shared_service_maria_primary }}-restore"
|
|
groups:
|
|
- "stage_{{ stage }}"
|
|
- 'restore'
|
|
when:
|
|
- database_engine in ['maria']
|
|
|
|
- name: "Add 'backup' servers to hosts if necessary"
|
|
add_host:
|
|
name: "{{ stage }}-backup-01"
|
|
groups:
|
|
- "stage_{{ stage }}"
|
|
- backup
|
|
changed_when: False
|
|
|
|
#############################################################
|
|
# Create restore server(s)
|
|
#############################################################
|
|
|
|
- hosts: "restore"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
gather_facts: false
|
|
remote_user: root
|
|
|
|
roles:
|
|
- role: hetzner-ansible-hcloud
|
|
vars:
|
|
sma_digitalocean_ttl: 60 # set it to 60sec to reduce DNS caching problems with internal IT in case of debugging ansible problems ;)
|
|
|
|
- role: hetzner-ansible-dns
|
|
vars:
|
|
record_data: "{{ stage_server_ip }}"
|
|
record_name: "{{ inventory_hostname }}"
|
|
tags:
|
|
- update_dns
|
|
|
|
#############################################################
|
|
# Provisioning server(s) for created inventory
|
|
#############################################################
|
|
|
|
- hosts: "restore"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
remote_user: root
|
|
vars:
|
|
ansible_ssh_host: "{{ stage_server_domain }}"
|
|
|
|
pre_tasks:
|
|
- name: "Import autodiscover pre-tasks"
|
|
import_tasks: tasks/autodiscover_pre_tasks.yml
|
|
become: false
|
|
tags:
|
|
- always
|
|
|
|
roles:
|
|
- role: hetzner-ansible-common
|
|
|
|
- role: devsec.hardening.ssh_hardening
|
|
tags:
|
|
- ssh_hardening
|
|
|
|
- role: hetzner-ansible-filebeat
|
|
when: filebeat_enabled | default(True)
|
|
|
|
- role: hetzner-ansible-node-exporter
|
|
when: node_exporter_enabled | default(True)
|
|
|
|
- role: restore_{{ database_engine }}
|
|
|
|
#############################################################
|
|
# add restore specific firewall rule
|
|
#############################################################
|
|
|
|
- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
gather_facts: false
|
|
connection: local
|
|
vars:
|
|
hcloud_firewall_objects_backup:
|
|
-
|
|
name: "{{ stage }}-restore-ssh-access"
|
|
state: present
|
|
rules:
|
|
-
|
|
direction: in
|
|
protocol: tcp
|
|
port: '22'
|
|
source_ips:
|
|
- "{{ lookup('community.general.dig', groups['backup'][0] + '.' + domain ) }}/32"
|
|
destination_ips: []
|
|
description: null
|
|
apply_to:
|
|
-
|
|
type: label_selector
|
|
label_selector:
|
|
selector: 'service=restore'
|
|
|
|
tasks:
|
|
- name: "Add hcloud firewall rule(s)"
|
|
include_role:
|
|
name: hetzner-ansible-hcloud
|
|
tasks_from: configure-firewall2
|
|
loop: "{{ hcloud_firewall_objects_backup }}"
|
|
loop_control:
|
|
loop_var: firewall_object
|
|
|
|
#############################################################
|
|
# Syncing backups from backup server to restore server
|
|
#############################################################
|
|
|
|
- hosts: "backup"
|
|
serial: "{{ serial_number | default(5) }}"
|
|
gather_facts: false
|
|
vars:
|
|
backupserver_system_user: 'backuphamster'
|
|
ansible_ssh_host: "{{ stage_server_domain }}"
|
|
tasks:
|
|
# I could not get it up and running with <synchronize> module
|
|
# to sync data from remote server A to remote server B
|
|
- name: "Syncing remote backups"
|
|
become: yes
|
|
become_user: '{{ backupserver_system_user }}'
|
|
vars:
|
|
database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}"
|
|
shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
|
|
|
|
#############################################################
|
|
# Restoring from backup
|
|
#############################################################
|
|
|
|
- hosts: "restore"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
gather_facts: false
|
|
vars:
|
|
ansible_ssh_host: "{{ stage_server_domain }}"
|
|
tasks:
|
|
- name: "Triggering restore"
|
|
become: yes
|
|
shell: '/root/restore.sh {{ stage }}'
|
|
|
|
- name: "Check for test data on postgres"
|
|
block:
|
|
|
|
- name: "Querying postgres ..."
|
|
become: yes
|
|
become_user: postgres
|
|
community.postgresql.postgresql_query:
|
|
db: dummytestdb
|
|
query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
|
|
named_args:
|
|
quote_val: 'Shall we play'
|
|
register: query_output
|
|
|
|
- assert:
|
|
that:
|
|
- 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
|
|
|
|
when:
|
|
- database_engine == 'postgres'
|
|
|
|
- name: "Check for test data on mariadb"
|
|
block:
|
|
|
|
- name: "Querying mariadb ..."
|
|
become: yes
|
|
become_user: root
|
|
community.mysql.mysql_query:
|
|
login_unix_socket: /run/mysqld/mysqld.sock
|
|
login_db: dummytestdb
|
|
query: SELECT movie FROM movie_quotes WHERE quote = %s
|
|
positional_args:
|
|
- 'Shall we play'
|
|
register: query_output
|
|
|
|
- assert:
|
|
that:
|
|
- 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
|
|
|
|
when:
|
|
- database_engine == 'maria'
|
|
|
|
|
|
#############################################################
|
|
# Deleting volumes for created inventory
|
|
#############################################################
|
|
- hosts: "restore"
|
|
serial: "{{ serial_number | default(5) }}"
|
|
gather_facts: false
|
|
|
|
tasks:
|
|
- name: "Delete volumes for <{{ inventory_hostname }}>"
|
|
include_role:
|
|
name: lvm_with_hetzner_volumes
|
|
tasks_from: _remove_hetzner_volumes
|
|
|
|
#############################################################
|
|
# Deleting servers/domains for created inventory
|
|
#############################################################
|
|
|
|
- hosts: "restore"
|
|
serial: "{{ serial_number | default(5) }}"
|
|
gather_facts: false
|
|
|
|
tasks:
|
|
- name: "Delete server <{{ inventory_hostname }}>"
|
|
include_role:
|
|
name: hetzner-ansible-hcloud
|
|
tasks_from: _set_server_state
|
|
vars:
|
|
- server_state: "absent"
|
|
|
|
- name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
|
|
include_role:
|
|
name: hetzner-ansible-dns
|
|
tasks_from: _remove_dns
|
|
vars:
|
|
record_to_remove: '{{ inventory_hostname }}'
|
|
|
|
#############################################################
|
|
# Sending metric to prometheus push gateway
|
|
#############################################################
|
|
|
|
- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
gather_facts: false
|
|
connection: local
|
|
run_once: true
|
|
vars:
|
|
prometheus_pushgw_host: prometheus-pushgateway.monitoring:9091 # pushgw-endpoint within k8s cluster running AWX
|
|
tasks:
|
|
# attention: do not remove manually set newline char at the end of metric line
|
|
# in case of removing: request will die with HTTP 400 bad request
|
|
- name: "Set Fact"
|
|
set_fact:
|
|
metric: "nightly_restore_successful_generic {{ lookup('pipe', 'date +%s') }}\n"
|
|
|
|
# body => trim needed to remove new line char
|
|
# due to created yaml string
|
|
- name: "Send metric to prometheus pushgw"
|
|
uri:
|
|
url: "http://{{ prometheus_pushgw_host }}/metrics/job/restore_test/creator/awx-run/database_engine/{{ database_engine }}"
|
|
method: POST
|
|
body: "{{ metric }}"
|
|
status_code: [200]
|
|
register: send_prome_metric
|
|
retries: 5
|
|
delay: 5
|
|
until: send_prome_metric.status in [200]
|
|
|
|
#############################################################
|
|
# Sending smardigo management message to process
|
|
#############################################################
|
|
|
|
- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
|
|
serial: "{{ serial_number | default(1) }}"
|
|
gather_facts: false
|
|
connection: local
|
|
run_once: true
|
|
vars:
|
|
connect_jwt_username: "{{ management_admin_username }}"
|
|
|
|
tasks:
|
|
- name: "Sending smardigo management message to <{{ shared_service_url_management }}>"
|
|
include_tasks: tasks/smardigo_management_message.yml
|