feat: added backups to smardigo portal

ansible-builder/README.md

@@ -0,0 +1,4 @@
+# Execution Environment for AWX
+
+    ansible-builder build --tag dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee:latest
+    docker push dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee

ansible-builder/context/.gitignore

@@ -0,0 +1 @@
+/_build/

ansible-builder/context/_build/bindep.txt

@@ -1,4 +0,0 @@
-python38-devel [platform:rpm compile]
-subversion [platform:rpm]
-subversion [platform:dpkg]
-git-lfs [platform:rpm]

ansible-builder/context/_build/requirements.txt

@@ -1,4 +0,0 @@
-hcloud
-urllib3
-jmespath
-git+https://github.com/ansible/ansible-builder.git@devel#egg=ansible-builder

ansible-builder/context/_build/requirements.yml

@@ -1,6 +0,0 @@
----
-collections:
-  - ansible.posix
-  - hetzner.hcloud
-  - community.general
-  - community.mysql

ansible-builder/execution-environment.yml

@@ -1,11 +1,10 @@
 ---
 version: 1
 dependencies:
-  galaxy: requirements.yml
-  python: requirements.txt
+  galaxy: ../galaxy-requirements.yml
+  python: ../pip-requirements
   system: bindep.txt
 
-
 additional_build_steps:
   append:
     - RUN alternatives --set python /usr/bin/python3
@@ -15,4 +14,3 @@ additional_build_steps:
     - CMD /run.sh
     - USER 1000 
     - RUN git lfs install
-

ansible-builder/requirements.txt

@@ -1,4 +0,0 @@
-hcloud
-urllib3
-jmespath
-git+https://github.com/ansible/ansible-builder.git@devel#egg=ansible-builder

ansible-builder/requirements.yml

@@ -1,6 +0,0 @@
----
-collections:
-  - ansible.posix
-  - hetzner.hcloud
-  - community.general
-  - community.mysql

create-database-backup.yml

@@ -3,7 +3,7 @@
 # creates database backup
 #   - postgres
 #     - executed on stage specific server: {{ stage }}-postgres-01
-#     - creates database backup for specifix database
+#     - creates database backup for specific database
 
 # Parameters:
 #   playbook inventory
@@ -23,8 +23,8 @@
 #############################################################
 
 - hosts: localhost
-  gather_facts: false
   connection: local
+  gather_facts: false
 
   pre_tasks:
     - name: "Check if ansible version is at least 2.10.x"
@@ -34,8 +34,11 @@
           - ansible_version.minor >= 10
         msg: "The ansible version has to be at least ({{ ansible_version.full }})"
 
-    - set_fact:
+    - name: "Parsing cluster_services_str into cluster_services"
+      set_fact:
         cluster_services: "{{ cluster_services_str | split(',') }}"
+      tags:
+        - always
 
   tasks:
     - name: "Add postgres servers to hosts if necessary"
@@ -57,42 +60,25 @@
   remote_user: root
   vars:
     ansible_ssh_host: "{{ stage_server_domain }}"
+    postgres_backup_state: dump
 
   roles:
     - role: connect-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'connect' in group_names"
 
     - role: gitea-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'gitea' in group_names"
 
     - role: keycloak-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'keycloak' in group_names"
 
     - role: webdav-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'webdav' in group_names"
 
     - role: workflow-index-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'workflow_index' in group_names"
 
     - role: workflow-proxy-postgres
-      vars:
-        database_create: False
-        database_backup: True
       when: "'workflow_proxy' in group_names"
 
 #############################################################

create-service.yml

@@ -44,20 +44,7 @@
       changed_when: False
 #      with_sequence: start=1 end={{ cluster_count | default(1) }}
       with_items: "{{ cluster_services }}"
-      when: item in ['connect']
-
-    - name: Add hosts
-      add_host:
-        name: "{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-01"
-#        name: "{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-{{ '%02d' | format(item|int) }}"
-        groups:
-        - "stage_{{ stage }}"
-        - "{{ cluster_service }}"
-        - "{{ item }}"
-      changed_when: False
-#      with_sequence: start=1 end={{ cluster_count | default(1) }}
-      with_items: "{{ cluster_services }}"
-      when: item in ['connect_wordpress']
+      when: item in ['connect', 'connect_wordpress']
 
 #############################################################
 # Setup services for created inventory

galaxy-requirements.yml

@@ -10,3 +10,4 @@ collections:
 - name: community.general
 - name: kubernetes.core
 - name: community.mysql
+- name: community.postgresql

restore-database-backup.yml

@@ -23,8 +23,8 @@
 #############################################################
 
 - hosts: localhost
-  gather_facts: false
   connection: local
+  gather_facts: false
 
   pre_tasks:
     - name: "Check if ansible version is at least 2.10.x"
@@ -34,8 +34,11 @@
           - ansible_version.minor >= 10
         msg: "The ansible version has to be at least ({{ ansible_version.full }})"
 
-    - set_fact:
+    - name: "Parsing cluster_services_str into cluster_services"
+      set_fact:
         cluster_services: "{{ cluster_services_str | split(',') }}"
+      tags:
+        - always
 
   tasks:
     - name: "Add postgres servers to hosts if necessary"
@@ -57,48 +60,25 @@
   remote_user: root
   vars:
     ansible_ssh_host: "{{ stage_server_domain }}"
+    postgres_backup_state: restore
 
   roles:
     - role: connect-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'connect' in group_names"
 
     - role: gitea-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'gitea' in group_names"
 
     - role: keycloak-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'keycloak' in group_names"
 
     - role: webdav-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'webdav' in group_names"
 
     - role: workflow-index-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'workflow_index' in group_names"
 
     - role: workflow-proxy-postgres
-      vars:
-        database_create: False
-        database_backup: False
-        database_restore: True
       when: "'workflow_proxy' in group_names"
 
 #############################################################

roles/awx/defaults/main.yml

@@ -23,3 +23,17 @@ kubernetes_awx_service_targetPort: "80"
 
 awx_ansible_username: "ansible"
 awx_ansible_password: "ansible"
+
+awx_job_templates:
+  - name: "create-database"
+  - name: "create-database-backup"
+  - name: "create-realm"
+  - name: "create-server"
+  - name: "create-service"
+  - name: "import-database"
+  - name: "remove-database"
+  - name: "remove-realm"
+  - name: "remove-server"
+  - name: "remove-service"
+  - name: "restore-database-backup"
+  - name: "update-monitoring"

roles/awx/tasks/awx-config-job-template.yml

@@ -69,7 +69,7 @@
     - awx_config
 
 - include_tasks: awx-config-job-template-credential.yml
-  loop: "{{ job.credentials }}"
+  loop: "{{ job_templates_credentials }}"
   loop_control:
     loop_var: awx_credential_id
   when: awx_job_template_id is defined

roles/awx/tasks/awx-config.yml

@@ -601,67 +601,7 @@
 
 - name: "Create job templates"
   include_tasks: awx-config-job-template.yml
-  loop:
-    - {
-        name: "create-database",
-        #description: "create-database",
-        #playbook_file: "create-database.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "create-realm",
-        #description: "create-realm",
-        #playbook_file: "create-realm.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "create-server",
-        #description: "create-server",
-        #playbook_file: "create-server.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "create-service",
-        #description: "create-service",
-        #playbook_file: "create-service.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "import-database",
-        #description: "import-database",
-        #playbook_file: "import-database.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "remove-database",
-        #description: "remove-database",
-        #playbook_file: "remove-database.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "remove-realm",
-        #description: "remove-realm",
-        #playbook_file: "remove-realm.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "remove-server",
-        #description: "remove-server",
-        #playbook_file: "remove-server.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "remove-service",
-        #description: "remove-service",
-        #playbook_file: "remove-service.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
-    - {
-        name: "update-monitoring",
-        #description: "update-monitoring",
-        #playbook_file: "update-monitoring.yml",
-        credentials: "{{ job_templates_credentials }}"
-      }
+  loop: "{{ awx_job_templates | flatten(levels=1) }}"
   loop_control:
     loop_var: job
   tags:

roles/connect-postgres/defaults/main.yml

@@ -4,7 +4,3 @@ postgres_acls:
   - name: "{{ connect_postgres_database }}"
     password: "{{ connect_postgres_password }}"
     trusted_cidr_entry: "{{ shared_service_network }}"
-
-database_create: True
-database_backup: False
-database_restore: False

roles/connect-postgres/tasks/main.yml

@@ -8,18 +8,12 @@
     name: postgres
     tasks_from: _postgres-acls
   when:
-  - database_create
+  - postgres_backup_state is not defined
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

roles/gitea-postgres/tasks/main.yml

@@ -8,18 +8,12 @@
     name: postgres
     tasks_from: _postgres-acls
   when:
-  - database_create
+  - postgres_backup_state is not defined
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

roles/keycloak-postgres/tasks/main.yml

@@ -10,16 +10,10 @@
   when:
   - database_create
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

roles/postgres/defaults/main.yml

@@ -1,15 +1,19 @@
 ---
 default_postgres_version: 13
-default_postgres_target_distribution: focal-pgdg # (bionic-pgdg, bullseye-pgdg, buster-pgdg,
-                                          #  focal-pgdg, groovy-pgdg, hirsute-pgdg,
-                                          #  sid-pgdg, stretch-pgdg, xenial-pgdg)
+# distributions:
+#   bionic-pgdg
+#   bullseye-pgdg
+#   buster-pgdg
+#   focal-pgdg
+#   groovy-pgdg
+#   hirsute-pgdg
+#   sid-pgdg
+#   stretch-pgdg
+#   xenial-pgdg
+default_postgres_target_distribution: focal-pgdg
 default_max_connections: 1000
 default_shared_buffers: 256MB
 
-postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_private_server_ip }}'"
-
 database_state: present
 
-database_create: True
-database_backup: False
-database_restore: False
+postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_private_server_ip }}'"

roles/postgres/tasks/_create-backup.yml

@@ -1,19 +0,0 @@
----
-- set_fact:
-    backup_path: "/backups"
-
-- name: "Create backup directory"
-  file:
-    path: "{{ backup_path }}"
-    state: directory
-    owner: "postgres"
-    group: "postgres"
-
-- name: "Creating backup ${dbname}__{{ custom_backup_name }}.sql ..."
-  become: true
-  become_user: postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    state: dump
-    target: "{{ backup_path }}/{{ item.name }}__{{ custom_backup_name }}.sql"
-  loop: "{{ postgres_acls }}"

roles/postgres/tasks/_postgres-backups.yml

@@ -0,0 +1,16 @@
+---
+- name: "Creating backup directory <{{ backup_directory }}>"
+  ansible.builtin.file:
+    path: "{{ backup_directory }}"
+    state: directory
+    owner: "postgres"
+    group: "postgres"
+
+- name: "Handle backup state <{{ postgres_backup_state }}> with suffix <{{ custom_backup_name }}>"
+  community.postgresql.postgresql_db:
+    name: "{{ item.name }}"
+    state: "{{ postgres_backup_state }}"
+    target: "{{ backup_directory }}/{{ item.name }}__{{ custom_backup_name }}.sql.gz"
+  loop: "{{ postgres_acls }}"
+  become_user: postgres
+  become: true

roles/postgres/tasks/_restore-backup.yml

@@ -1,9 +0,0 @@
----
-- name: "Restoring backup ${dbname}__{{ custom_backup_name }}.sql  ..."
-  become: true
-  become_user: postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    state: restore
-    target: "{{ backup_directory }}/{{ item.name }}__{{ custom_backup_name }}.sql"
-  loop: "{{ postgres_acls }}"

roles/postgres/tasks/base-requirements.yml

@@ -69,15 +69,7 @@
     group: postgres
     mode: "g+s"
 
-- name: "Creating backups directory if necessary"
-  file:
-    state: directory
-    path: /backups
-    owner: postgres
-    group: postgres
-    mode: "g+s"
-
-- name: "Install prometheus postgres exporter .."
+- name: "Install prometheus postgres exporter..."
   apt:
     name: "prometheus-postgres-exporter"
     update_cache: yes

roles/postgres/tasks/base-requirements_backup.yml

@@ -1,7 +1,5 @@
 ---
-- name: creating some hcloud volumes for backup purpose
-  delegate_to: localhost
-  become: false
+- name: "Creating some hcloud volumes for backup purpose"
   hcloud_volume:
     api_token: "{{ hetzner_authentication_token }}" 
     name: "postgres-backup--{{ inventory_hostname }}--vol{{ item }}"
@@ -14,14 +12,16 @@
     delete_protection: yes
   loop: "{{ range(1,2) | list }}"
   register: created_volume
-
-- name: getting all hcloud volumes for
   delegate_to: localhost
   become: false
+
+- name: "Getting all hcloud volumes for"
   hcloud_volume_info:
     api_token: "{{ hetzner_authentication_token }}" 
     label_selector: "stage={{ stage }},used_for={{ inventory_hostname }}"
   register: hcloud_volumes_found 
+  delegate_to: localhost
+  become: false
 
 - set_fact:
     pvs: "{{ hcloud_volumes_found.hcloud_volume_info | json_query(jmesquery) }}"
@@ -30,14 +30,14 @@
   vars:
     jmesquery: "[*].linux_device"
 
-- name: Create a volume group on top of all found hcloud volumes
+- name: "Creating a volume group on top of all found hcloud volumes"
   community.general.lvg:
     vg: "{{ vg_name }}"
     pvs: "{{ pvs }}"
     pvresize: yes
   register: create_vg
 
-- name: Create logical volume
+- name: "Create logical volume"
   community.general.lvol:
     vg: "{{ vg_name }}"
     lv: "{{ lv_name }}" 
@@ -45,12 +45,12 @@
   when:
   - create_vg.changed
 
-- name: format volume
+- name: "Format volume"
   filesystem:
     fstype: ext4
     dev: "/dev/{{ vg_name }}/{{ lv_name }}"
 
-- name: mount created LVM volume
+- name: "Mount created LVM volume"
   mount:
     path: "{{ backup_directory }}"
     src: "/dev/{{ vg_name }}/{{ lv_name }}"

roles/postgres/tasks/main.yml

@@ -20,19 +20,3 @@
 - name: Include Slave Requirements
   include_tasks: slave-requirements.yml
   when: server_type == "slave"
-  when:
-  - database_create
-
-- name: "Create postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _create-backup
-  when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore

roles/postgres/tasks/slave-requirements.yml

@@ -37,14 +37,6 @@
     state: mounted
     fstype: nfs
 
-# - name: Mount backup NFS volume
-#   ansible.posix.mount:
-#     src: db-backups:/backups
-#     path: /backups
-#     opts: "rw,bg,hard,nfsvers=4.2,x-systemd.automount,x-systemd.requires=network-online.target,x-systemd.device-timeout=60 0 0"
-#     state: mounted
-#     fstype: nfs
-
 - name: Stop postgres on slave
   service:
     name: postgresql

roles/webdav-postgres/tasks/main.yml

@@ -7,18 +7,12 @@
     name: postgres
     tasks_from: _postgres-acls
   when:
-  - database_create
+  - postgres_backup_state is not defined
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

roles/workflow-index-postgres/tasks/main.yml

@@ -8,18 +8,12 @@
     name: postgres
     tasks_from: _postgres-acls
   when:
-  - database_create
+  - postgres_backup_state is not defined
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

roles/workflow-proxy-postgres/tasks/main.yml

@@ -8,18 +8,12 @@
     name: postgres
     tasks_from: _postgres-acls
   when:
-  - database_create
+  - postgres_backup_state is not defined
 
-- name: "Create postgres backup"
+- name: "Creating/restoring postgres backup"
   include_role:
     name: postgres
-    tasks_from: _create-backup
+    tasks_from: _postgres-backups.yml
   when:
-  - database_backup
-
-- name: "Restore postgres backup"
-  include_role:
-    name: postgres
-    tasks_from: _restore-backup
-  when:
-  - database_restore
+  - postgres_backup_state is defined
+  - postgres_backup_state in ['dump', 'restore']

smardigo/provisioning/process-search/simple-connect.json

@@ -50,9 +50,15 @@
       }, {
         "value" : "Service wird gestartet",
         "style" : "traffic-light-yellow"
+      }, {
+        "value" : "Service wird aktualisiert",
+        "style" : "traffic-light-yellow"
       }, {
         "value" : "Service gestartet",
         "style" : "traffic-light-green"
+      }, {
+        "value" : "Antrag abgebrochen",
+        "style" : "traffic-light-red"
       }, {
         "value" : "Service gelöscht",
         "style" : "traffic-light-red"

smardigo/provisioning/script/ansible-start.groovy

@@ -9,9 +9,11 @@ def env = [
   stage: cluster.stage,
   current_realm_name: tenant.key,
   current_realm_display_name: tenant.name,
-  database_backup_file: execution.getVariable('databaseBackupFilename'),
   tenant_id: tenant.key
 ]
+if (binding.hasVariable('extraVariables')) {
+  env << extraVariables
+}
 
 def ansibleCommand= 'ansible-playbook ' + smardigoManagementAction + '.yml --vault-password-file ~/vault-pass'
 def ansibleEnvironment= ' -e \"'

smardigo/provisioning/script/create-awx-paramaters.groovy

@@ -1,4 +1,4 @@
-[
+def env = [
   scope_id: contextScopeId,
   process_instance_id: execution.getProcessInstanceId(),
   smardigo_management_action: smardigoManagementAction,
@@ -9,6 +9,10 @@
   stage: cluster.stage,
   current_realm_name: tenant.key,
   current_realm_display_name: tenant.name,
-  database_backup_file: execution.getVariable('databaseBackupFilename'),
   tenant_id: tenant.key
 ]
+if (binding.hasVariable('extraVariables')) {
+  env << extraVariables
+}
+
+env