Feature/dev 1110 Extend backup process to support multiple db servers per stage

create-remote-database-backup.yml

@@ -1,5 +1,4 @@
 ---
-
 #    ******         **         ******    **   **   **     **   *******    #
 #   /*////**       ****       **////**  /**  **   /**    /**  /**////**   #
 #   /*   /**      **//**     **    //   /** **    /**    /**  /**   /**   #
@@ -35,12 +34,12 @@
   gather_facts: false
 
   tasks:
-#     add virtual server to load stage specific variables as context
+    #     add virtual server to load stage specific variables as context
     - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
       add_host:
         name: "{{ stage }}-virtual-host-to-read-groups-vars"
         groups:
-        - "stage_{{ stage }}"
+          - "stage_{{ stage }}"
       changed_when: False
 
 - hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
@@ -55,31 +54,32 @@
       tags:
         - always
 
-  tasks:
-    - name: "Add {{ database_engine }} servers to hosts if necessary"
-      add_host:
-        name: "{{ shared_service_postgres_secondary }}"
-        groups:
-        - "stage_{{ stage }}"
-        - '{{ database_engine }}'
-      when:
-        - database_engine in ['postgres']
+    - name: "Import autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      become: false
+      tags:
+        - always
 
+  tasks:
     - name: "Add {{ database_engine }} servers to hosts if necessary"
       add_host:
-        name: "{{ shared_service_maria_primary }}"
+        name: "{{ item.name }}"
         groups:
-        - "stage_{{ stage }}"
-        - '{{ database_engine }}'
+          - "stage_{{ stage }}"
+          - "{{ database_engine }}"
       when:
-        - database_engine in ['maria']
+        - (database_engine == 'postgres' and item.service == 'postgres' and (item.role | default('')) == 'slave')
+          or (database_engine == 'maria' and item.service == 'maria')
+      loop: "{{ stage_server_infos }}"
 
-    - name: "Add 'storage' servers to hosts if necessary"
+    - name: "Add 'backup' servers to hosts if necessary"
       add_host:
         name: "{{ stage }}-backup-01"
         groups:
-        - "stage_{{ stage }}"
-        - "storage"
+          - "stage_{{ stage }}"
+          - "backup"
+      when:
+        - "'postgres' in groups or 'maria' in groups"
 
 ##############################################################
 ##   Creating remote database backups for created inventory
@@ -90,45 +90,44 @@
   gather_facts: false
   vars:
     ansible_ssh_host: "{{ stage_server_domain }}"
-    current_date_time: '{{ get_current_date_time }}'
+    current_date_time: "{{ get_current_date_time }}"
 
   tasks:
     - name: "Trigger backup mechanism"
       include_role:
-        name: '{{ database_engine }}'
+        name: "{{ database_engine }}"
         tasks_from: _create_backup
 
 #############################################################
-#   Syncing remote database backups to storage server
+#   Syncing remote database backups to backup server
 #############################################################
 
-- hosts: "postgres:maria:storage"
+- hosts: "postgres:maria:backup"
   serial: "{{ serial_number | default(5) }}"
   gather_facts: false
   vars:
     ansible_ssh_host: "{{ stage_server_domain }}"
-    storageserver_system_user: 'backuphamster'
+    backup_server_system_user: "backuphamster"
 
   tasks:
     # I could not get it up and running with <synchronize> module
     # to sync data from remote server A to remote server B
     - name: "Syncing remote backups"
       become: yes
-      become_user: '{{ storageserver_system_user }}'
-      vars:
-        # should work with non-fqdn due to existing entry in /etc/hosts
-        database_server_ip: "{{ shared_service_postgres_secondary if database_engine == 'postgres' else shared_service_maria_primary }}"
-      shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+      become_user: "{{ backup_server_system_user }}"
+      shell: "/home/{{ backup_server_system_user }}/pull_remote_backups.sh {{ item }} {{ stage }} {{ database_engine }}"
+      with_items: "{{ (groups['postgres'] | default([])) + (groups['maria'] | default([])) }}"
       when:
-        - inventory_hostname in groups['storage']
+        - inventory_hostname in groups['backup']
 
     - name: "Cleanup remote backup dirs: {{ database_engine }}"
       become: yes
       file:
-        path: '{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}'
+        path: "{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}"
         state: absent
       when:
-        - not inventory_hostname in groups['storage']
+        - not inventory_hostname in groups['backup']
+        - inventory_hostname in groups [database_engine]
 
 #############################################################
 #   Sending smardigo management message to process

group_vars/all/database.yml

@@ -5,13 +5,13 @@ shared_service_postgres_primary: "{{ stage }}-postgres-01"
 shared_service_postgres_secondary: "{{ stage }}-postgres-02"
 
 shared_service_pg_master_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', shared_service_postgres_primary )
+  | selectattr('name', '==', shared_service_postgres_primary )
   | map(attribute='private_ip')
   | list
   | first
   | default('-') }}"
 shared_service_pg_slave_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', shared_service_postgres_secondary )
+  | selectattr('name', '==', shared_service_postgres_secondary )
   | map(attribute='private_ip')
   | list
   | first

group_vars/postgres01/plain.yml

@@ -1,13 +1,3 @@
 ---
-shared_service_pg_master_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', stage + '-postgres-01' )
-  | map(attribute='private_ip')
-  | list
-  | first
-  | default('-') }}"
-shared_service_pg_slave_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', stage + '-postgres-02' )
-  | map(attribute='private_ip')
-  | list
-  | first
-  | default('-') }}"
+shared_service_postgres_primary: "{{ stage }}-postgres-01"
+shared_service_postgres_secondary: "{{ stage }}-postgres-02"

group_vars/postgres02/plain.yml

@@ -1,13 +1,3 @@
 ---
-shared_service_pg_master_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', stage + '-postgres02-01' )
-  | map(attribute='private_ip')
-  | list
-  | first
-  | default('-') }}"
-shared_service_pg_slave_ip: "{{ stage_server_infos
-  | selectattr('name', 'match', stage + '-postgres02-02' )
-  | map(attribute='private_ip')
-  | list
-  | first
-  | default('-') }}"
+shared_service_postgres_primary: "{{ stage }}-postgres02-01"
+shared_service_postgres_secondary: "{{ stage }}-postgres02-02"

inventory_plugins/netgo-hcloud.py

@@ -222,8 +222,8 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             serverId = server["id"]
             serverName = server["name"]
             serverLabels = server["labels"]
-            serverStage = serverLabels["stage"]
-            serverService = serverLabels["service"]
+            serverStage = serverLabels.get("stage", "this_stage_does_not_exist")
+            serverService = serverLabels.get("service", "this_group_does_not_exist")
             serverPublicIp = server["public_net"]["ipv4"]["ip"]
             serverPrivateIp = '-'
 

restore-remote-database-backup.yml

@@ -1,5 +1,4 @@
 ---
-
 # restores remote database backup
 #   - postgres
 #     - executed on stage specific server: {{ shared_service_postgres_primary }}-restore
@@ -40,32 +39,46 @@
         - "stage_{{ stage }}"
       changed_when: False
 
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+
+  pre_tasks:
+    - name: "Import autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      become: false
+      tags:
+        - always
+
   tasks:
-    - name: "Add {{ database_engine }}-restore servers to hosts if necessary"
-      add_host:
-        name: "{{ shared_service_postgres_secondary }}-restore"
-        groups:
-        - "stage_{{ stage }}"
-        - 'restore'
+    - name: "Get all {{ database_engine }} servers in stage {{ stage }}"
+      set_fact:
+        database_servers: "{{ database_servers | default([]) + [item.name] }}"
       when:
-        - database_engine in ['postgres']
+        - (database_engine == 'postgres' and item.service == 'postgres' and (item.role | default('')) == 'slave')
+          or (database_engine == 'maria' and item.service == 'maria')
+      loop: "{{ stage_server_infos }}"
 
-    - name: "Add {{ database_engine }}-restore servers to hosts if necessary"
+    - name: "Add {{ database_engine }} restore server to hosts if necessary"
       add_host:
-        name: "{{ shared_service_maria_primary }}-restore"
+        name: "{{ stage }}-{{ database_engine }}-restore"
         groups:
-        - "stage_{{ stage }}"
-        - 'restore'
+          - "stage_{{ stage }}"
+          - "restore"
       when:
-        - database_engine in ['maria']
+        - (database_engine == 'postgres')
+          or (database_engine == 'maria')
 
     - name: "Add 'backup' servers to hosts if necessary"
       add_host:
         name: "{{ stage }}-backup-01"
         groups:
-        - "stage_{{ stage }}"
-        - backup
-      changed_when: False
+          - "stage_{{ stage }}"
+          - "backup"
+      when:
+        - (database_engine == 'postgres')
+          or (database_engine == 'maria')
 
 #############################################################
 #   Create restore server(s)
@@ -165,17 +178,23 @@
   serial: "{{ serial_number | default(5) }}"
   gather_facts: false
   vars:
-    backupserver_system_user: 'backuphamster'
+    backup_server_system_user: 'backuphamster'
     ansible_ssh_host: "{{ stage_server_domain }}"
+    database_servers: "{{ hostvars[stage + '-virtual-host-to-read-groups-vars'].database_servers }}"
   tasks:
     # I could not get it up and running with <synchronize> module
     # to sync data from remote server A to remote server B
     - name: "Syncing remote backups"
       become: yes
-      become_user: '{{ backupserver_system_user }}'
+      become_user: '{{ backup_server_system_user }}'
       vars:
-        database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}"
-      shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+        restore_server: "{{ groups['restore'][0] }}.{{ domain }}"
+      shell: '/home/{{ backup_server_system_user }}/push_backups_to_restore_server.sh {{ restore_server }} {{ database_server }} {{ stage }} {{ database_engine }}'
+      loop: "{{ database_servers }}"
+      loop_control:
+        loop_var: database_server
+      when:
+        - inventory_hostname in groups['backup']
 
 #############################################################
 #   Restoring from backup
@@ -186,52 +205,15 @@
   gather_facts: false
   vars:
     ansible_ssh_host: "{{ stage_server_domain }}"
+    database_servers: "{{ hostvars[stage + '-virtual-host-to-read-groups-vars'].database_servers }}"
   tasks:
-    - name: "Triggering restore"
-      become: yes
-      shell: '/root/restore.sh {{ stage }}'
-
-    - name: "Check for test data on postgres"
-      block:
-
-      - name: "Querying postgres ..."
-        become: yes
-        become_user: postgres
-        community.postgresql.postgresql_query:
-          db: dummytestdb
-          query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
-          named_args:
-            quote_val: 'Shall we play'
-        register: query_output
-
-      - assert:
-          that:
-          - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
-
-      when:
-        - database_engine == 'postgres'
-
-    - name: "Check for test data on mariadb"
-      block:
-
-      - name: "Querying mariadb ..."
-        become: yes
-        become_user: root
-        community.mysql.mysql_query:
-          login_unix_socket: /run/mysqld/mysqld.sock
-          login_db: dummytestdb
-          query: SELECT movie FROM movie_quotes WHERE quote = %s
-          positional_args:
-            - 'Shall we play'
-        register: query_output
-
-      - assert:
-          that:
-          - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
-
-      when:
-        - database_engine == 'maria'
-
+    - name: "Start restore test for '{{ database_engine }}' backup from server '{{ database_server }}'"
+      include_role:
+        name: "restore_{{ database_engine }}"
+        tasks_from: _restore_test
+      loop: "{{ database_servers }}"
+      loop_control:
+        loop_var: database_server
 
 #############################################################
 #   Deleting volumes for created inventory

roles/backup/files/pull_remote_backups.sh

@@ -10,22 +10,22 @@ set -euo pipefail
 DATE=$(date +%F)
 DATE_TIME=$(date +%F_%H:%M)
 REMOTE_SYSTEM_USER=backupuser
-DATABASE_SERVER_IP=$1
+DATABASE_SERVER=$1
 STAGE=$2
 DATABASE_ENGINE=$3
-DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}
+DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}
 METRICS_FILE=${HOME}/backup_status_${DATABASE_ENGINE}.prom
 LOG_FILE=${DEST_DIR}/backup_${DATE_TIME}.log
 
+# Create backup directory ${DEST_DIR} if not exist
+mkdir -p ${DEST_DIR}
+
 # Redirect stderr to stdout and save everything to log file
 exec > ${LOG_FILE} 2>&1
 
 # Log backup sync start time
 echo "----- Start backup Sync - ${DATE_TIME} -----"
 
-# Create backup directory ${DEST_DIR} if not exist
-mkdir -p ${DEST_DIR}
-
 # Remove files oder than 48h in ${DEST_DIR}
 find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR"
 [ "$?" != "0" ] && exit 1
@@ -33,8 +33,8 @@ find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR"
 echo "Removing logfiles older than 7d ..."
 find $DEST_DIR -type f -mtime +7 -name "backup_*.log" -print0 | xargs -I OLD_FILES -0 rm -rf "OLD_FILES"
 
-# Start rsync job from ${DATABASE_SERVER_IP} to ${DEST_DIR}/
-rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/
+# Start rsync job from ${DATABASE_SERVER} to ${DEST_DIR}/
+rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/
 [ "$?" -eq "0" ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1"
 
 BACKUP_STATUS_FILE=$(ls -t1 ${DEST_DIR}/${DATE}/backup_finished_${DATE}_* | head -n1)

roles/backup/files/push_backups_to_restore_server.sh

@@ -4,17 +4,18 @@
 #
 
 REMOTE_SYSTEM_USER=backupuser
-DATABASE_SERVER_IP=$1
-STAGE=$2
-DATABASE_ENGINE=$3
+RESTORE_SERVER=$1
+DATABASE_SERVER=$2
+STAGE=$3
+DATABASE_ENGINE=$4
 
 # currently it defaults to todays date
 DATE=$(date +%F)
 
-LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}"
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}"
 BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1)
 
-REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}"
+REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}"
 DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"
 
 if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then
@@ -22,16 +23,16 @@ if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then
 fi
 
 # avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process
-ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${DATABASE_SERVER_IP}
+ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${RESTORE_SERVER}
 
 SSH_OPTIONS='-o StrictHostKeyChecking=no'
 
 # needed due to unknown rsync option --mkpath in rsync version 3.1.3
-ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}"
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "mkdir -p ${DEST_DIR}"
 
-rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR}
+rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER}:${DEST_DIR}
 
 BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}')
 
-ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}"
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}"
 

roles/lvm_with_hetzner_volumes/tasks/main.yml

@@ -69,12 +69,12 @@
   when: create_vg.changed
 
 - name: "Format volume"
-  community.general.system.filesystem:
+  community.general.filesystem:
     fstype: ext4
     dev: "/dev/{{ vg_name }}/{{ lv_name }}"
 
 - name: "Resize volume" # noqa no-handler
-  community.general.system.filesystem:
+  community.general.filesystem:
     fstype: ext4
     dev: "/dev/{{ vg_name }}/{{ lv_name }}"
     resizefs: true

roles/postgres/tasks/master-requirements.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Check role exists # noqa no-changed-when
   become: true
   become_user: postgres
@@ -181,9 +180,7 @@
   register: cmd_ret
   changed_when: cmd_ret.rc != 0
 
-# only needed in case of install from scratch
 - name: "Ensure test db stuff"
-  when: postgres_ensure_testdb | default(False)
   block:
     - name: "Copy testdb.sql to ensure test DB"
       ansible.builtin.copy:
@@ -195,6 +192,13 @@
       loop:
         - testdb.sql
 
+    - name: "Drop test DB if exists"
+      become: true
+      become_user: postgres
+      community.postgresql.postgresql_db:
+        name: dummytestdb
+        state: absent
+
     - name: "Ensure test DB"
       become: true
       become_user: postgres

roles/restore_maria/files/restore.sh

@@ -5,18 +5,18 @@
 #
 
 STAGE=$1
+DATABASE_SERVER=$2
 
 DATADIR='/var/lib/mysql'
 DATE=$(date +%F)
 
-
 systemctl stop mariadb
 
 rm -rf ${DATADIR}_moved
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}
 
-LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria"
+LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/maria/${DATABASE_SERVER}"
 BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
 
 # --batch  => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" <<

roles/restore_maria/tasks/_restore_test.yml

@@ -0,0 +1,25 @@
+---
+- name: "Triggering restore" # noqa command-instead-of-shell no-changed-when
+  become: true
+  shell: "/root/restore.sh {{ stage }} {{ database_server }}"
+
+- name: "Check for test data on mariadb"
+  block:
+    - name: "Querying mariadb ..."
+      become: true
+      become_user: root
+      community.mysql.mysql_query:
+        login_unix_socket: /run/mysqld/mysqld.sock
+        login_db: dummytestdb
+        query: SELECT movie FROM movie_quotes WHERE quote = %s
+        positional_args:
+          - "Shall we play"
+      register: query_output
+
+    - name: "Compare query result with given parameters ..."
+      assert:
+        that:
+          - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
+
+  when:
+    - database_engine == 'maria'

roles/restore_postgres/files/restore.sh

@@ -4,6 +4,7 @@
 #
 
 STAGE=$1
+DATABASE_SERVER=$2
 
 DATADIR='/var/lib/postgresql/13/main'
 DATE=$(date +%F)
@@ -18,7 +19,7 @@ rm -rf ${DATADIR}_moved
 mv ${DATADIR} ${DATADIR}_moved
 mkdir -p ${DATADIR}
 
-LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres"
+LOCAL_BACKUP_DIR="/home/backupuser/backups/${STAGE}/postgres/${DATABASE_SERVER}"
 BACKUP_FILE_ENCRYPTED=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
 
 # --batch  => avoid error: >> gpg: cannot open '/dev/tty': No such device or address" <<

roles/restore_postgres/tasks/_restore_test.yml

@@ -0,0 +1,24 @@
+---
+- name: "Triggering restore" # noqa command-instead-of-shell no-changed-when
+  become: true
+  shell: "/root/restore.sh {{ stage }} {{ database_server }}"
+
+- name: "Check for test data on postgres"
+  block:
+    - name: "Querying postgres ..."
+      become: true
+      become_user: postgres
+      community.postgresql.postgresql_query:
+        db: dummytestdb
+        query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
+        named_args:
+          quote_val: "Shall we play"
+      register: query_output
+
+    - name: "Compare query result with given parameters ..."
+      assert:
+        that:
+          - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
+
+  when:
+    - database_engine == 'postgres'

tasks/autodiscover_pre_tasks.yml

@@ -142,7 +142,6 @@
   tags:
     - always
 
-
 #############################################################
 #   Setting stage infos as facts
 #############################################################
@@ -151,21 +150,26 @@
 - name: "Reading hetzner server infos for stage <{{ stage }}/{{ stage_kube }}>"
   set_fact:
     stage_server_infos: "\
-      {% set  list= [] %}\
+      {% set stage_server_infos= [] %}\
       {% for server in hetzner_servers %}\
         {% if server.labels.manual is not defined %}\
+          {% set stage_server_info={'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id} %}\
+          {% for label in (server.labels | dict2items) %}\
+            {% set x=stage_server_info.__setitem__(label.key,label.value) %}\
+          {% endfor %}\
+
+          {% set x=stage_server_info.__setitem__('private_ip','-') %}\
           {% if (server.private_net|length) > 0 %}\
             {% for private_net in server.private_net %}\
               {% if private_net.network == stage_network_id|int %}\
-                {{ list.append({'service': server.labels.service | default(''), 'private_ip': private_net.ip, 'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id}) }}\
+                {% set x=stage_server_info.__setitem__('private_ip',private_net.ip) %}\
               {% endif %}\
             {% endfor %}\
-          {% else %}\
-            {{ list.append({'service': server.labels.service | default(''), 'private_ip': '-', 'public_ip': server.public_net.ipv4.ip, 'name': server.name, 'id': server.id}) }}\
           {% endif %}\
         {% endif %}\
+        {{ stage_server_infos.append(stage_server_info) }}\
       {% endfor %}\
-      {{ list|list }}"
+      {{ stage_server_infos | list }}"
   tags:
     - always
 
@@ -202,5 +206,6 @@
       - "ingess private ip: <{{ shared_service_kube_loadbalancer_public_ip }}>"
       - "logstash public ip: <{{ shared_service_loadbalancer_logstash_public_ip }}>"
       - "logstash private ip: <{{ shared_service_loadbalancer_logstash_private_ip }}>"
+      - "server info: <{{ stage_server_infos | selectattr('name', 'equalto', inventory_hostname) | default({}) }}>"
   tags:
     - always