diff --git a/galaxy-requirements.yml b/galaxy-requirements.yml index 586c6d8..027cfb7 100644 --- a/galaxy-requirements.yml +++ b/galaxy-requirements.yml @@ -24,12 +24,8 @@ roles: version: 3.3.2 collections: -#- name: hetzner.hcloud -# version: 1.6.0 - name: hetzner.hcloud - src: https://github.com/fredFromJupiter/hetzner.hcloud.git - github_branch: "fredFromJupiter-patch-1" - scm: git + version: 1.6.0 - name: community.general - name: community.docker version: 2.1.1 diff --git a/restore-remote-database-backup.yml b/restore-remote-database-backup.yml index e00554f..4a5fb62 100644 --- a/restore-remote-database-backup.yml +++ b/restore-remote-database-backup.yml @@ -68,6 +68,8 @@ roles: - role: hcloud + vars: + sma_digitalocean_ttl: 60 # set it to 60sec to reduce DNS caching problems with internal IT in case of debugging ansible problems ;) ############################################################# # Provisioning server(s) for created inventory diff --git a/roles/backup/files/push_backups_to_restore_server.sh b/roles/backup/files/push_backups_to_restore_server.sh index 9dd5010..80e959a 100644 --- a/roles/backup/files/push_backups_to_restore_server.sh +++ b/roles/backup/files/push_backups_to_restore_server.sh @@ -17,6 +17,9 @@ BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}" DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/" +# avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process +ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${DATABASE_SERVER_IP} + SSH_OPTIONS='-o StrictHostKeyChecking=no' # needed due to unknown rsync option --mkpath in rsync version 3.1.3 diff --git a/roles/hcloud/tasks/_set_server_state.yml b/roles/hcloud/tasks/_set_server_state.yml index 85ef4de..3af0778 100644 --- a/roles/hcloud/tasks/_set_server_state.yml +++ b/roles/hcloud/tasks/_set_server_state.yml @@ -1,57 +1,35 @@ --- - -- name: "Checking state for server <{{ inventory_hostname }}> is <{{ server_state }}>" - hetzner.hcloud.hcloud_server: - api_token: "{{ hetzner_authentication_ansible }}" - name: "{{ inventory_hostname }}" - server_type: "{{ hetzner_server_type }}" - image: "{{ hetzner_server_image }}" - ssh_keys: "{{ hetzner_ssh_keys }}" - labels: "{{ hetzner_server_labels }}" - location: nbg1 - state: "{{ server_state }}" - delegate_to: 127.0.0.1 - become: false - async: 300 - poll: 5 - register: hcloud_response - ignore_errors: yes - -# poweron server if server not running -- name: "Ensure Server is STARTED when server_state=present" - hetzner.hcloud.hcloud_server: - api_token: "{{ hetzner_authentication_ansible }}" - name: "{{ inventory_hostname }}" - state: "started" - delegate_to: 127.0.0.1 - become: false - async: 300 - poll: 15 - when: - - server_state == 'present' - - not hcloud_response.msg is defined - -# just catching hetzner error and trying to avoid manual interaction -- name: "DEBUGGING Block for hetzner << message not found >> - error" +- name: "Block to handle hetzner server state in case of problems" + vars: + max_retries: "{{ max_retries | default(15) | int }}" + retry_delay: "{{ retry_delay | default(60) | int }}" block: - - name: "DEBUG: hcloud_response" - debug: - msg: '{{ hcloud_response.msg }}' + - name: "Increment the retry count" + set_fact: + retry_count: "{{ retry_count | default(0) | int + 1 }}" - - name: "Ensure Server is STARTED when server_state=present" + - name: "Checking state for server <{{ inventory_hostname }}> is <{{ server_state }}>" hetzner.hcloud.hcloud_server: api_token: "{{ hetzner_authentication_ansible }}" name: "{{ inventory_hostname }}" - state: "started" + server_type: "{{ hetzner_server_type }}" + image: "{{ hetzner_server_image }}" + ssh_keys: "{{ hetzner_ssh_keys }}" + labels: "{{ hetzner_server_labels }}" + location: nbg1 + state: "{{ server_state }}" delegate_to: 127.0.0.1 become: false - retries: 40 - delay: 15 - until: server_locked.rc == 0 - register: server_locked + async: 300 + poll: 5 + register: hcloud_response + ignore_errors: yes + + - name: "Block - DEBUG: hcloud_response" + debug: + msg: '{{ hcloud_response.msg }}' when: - - server_state == 'present' - - hcloud_response.msg == 'cannot perform operation because server is locked' + - hcloud_response.msg is defined - name: "Ensure Server is STARTED when server_state=present" hetzner.hcloud.hcloud_server: @@ -60,10 +38,27 @@ state: "started" delegate_to: 127.0.0.1 become: false - async: 300 + async: 150 poll: 15 + register: hcloud_response when: - server_state == 'present' - when: - - hcloud_response.msg is defined + rescue: + - name: "RESCUE - fail: Maximum retries reached" + fail: + msg: "max_retries of {{ max_retries }} reached. Plz check." + when: retry_count | int == max_retries | int + + - name: "RESCUE-fail DEBUG: hcloud_response" + debug: + msg: '{{ hcloud_response.msg }}' + + - name: "RESCUE: wait_for between retries" + wait_for: + timeout: "{{ retry_delay }}" + delegate_to: localhost + become: false + + - name: "Include _set_server one time again - retry_count: {{ retry_count }}" + include_tasks: _set_server_state.yml diff --git a/roles/sma_digitalocean/tasks/domain.yml b/roles/sma_digitalocean/tasks/domain.yml index c909dcb..ca57bfc 100644 --- a/roles/sma_digitalocean/tasks/domain.yml +++ b/roles/sma_digitalocean/tasks/domain.yml @@ -68,7 +68,7 @@ vars: create_record: type: 'A' - ttl: 1800 + ttl: "{{ sma_digitalocean_ttl | default(1800) }}" data: "{{ record_data }}" name: "{{ record_name }}" when: diff --git a/smardigo/backup/process/primary-process.bpmn b/smardigo/backup/process/primary-process.bpmn index 67b6a5a..f649485 100644 --- a/smardigo/backup/process/primary-process.bpmn +++ b/smardigo/backup/process/primary-process.bpmn @@ -80,7 +80,7 @@ Flow_1a8vxf0 - 0 0 0 * * ? + 0 0 * * * ?