Merge branch 'cusqa_dev1029+debuggingpgbkp_nextlevel' into 'cusqa'

DEV-1029: cusqa - added script to retrigger backup in case

See merge request smardigo-hetzner/mobene/prodwork01-mobene-deployment!33

templates/cm_pg_backup_scripts.yaml

@@ -1,11 +1,40 @@
 apiVersion: v1
 data:
+  backup_retry.sh: |
+    #!/bin/bash
+    #
+    #
+    
+    BASENAME=$(basename $0)
+
+    function log
+    {
+        touch /tmp/${BASENAME}.log
+        echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*" >> /tmp/${BASENAME}.log
+    }
+
+    log "INFO - script was executed"
+  
+    LAST_BKP=$(envdir "/run/etc/wal-e.d/env" wal-g backup-list --detail --json | jq -r .[-1].finish_time)
+    
+    LAST_BKP_DATE_IN_UNIXSEC=$(date -d ${LAST_BKP} +"%s")
+    
+    NOW=$(date +%s -u)
+    
+    if [ $((NOW-LAST_BKP_DATE_IN_UNIXSEC)) -lt 86400 ]; then
+      log "INFO - last backup created within 24h. no backup rescheduling needed"
+      exit 0
+    else
+      log "INFO - last backup created older than 24h. triggering backup..."
+      envdir "/run/etc/wal-e.d/env" /scripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data"
+    fi
   postgres_backup.sh: |
     #!/bin/bash
 
     echo "Plz check DEBUG_LOG for debugging purpose. EVERY output will be redirected!"
     # fgoerz DEV-1029
     # pipe all output to file for debugging purpose
+    exec 3>&1
     exec 1>>/tmp/pg_backup_`date +%F`.log
     exec 2>&1
 
@@ -76,7 +105,7 @@ data:
     # push a new base backup
     log "producing a new backup"
     # We reduce the priority of the backup for CPU consumption
-    exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}"
+    exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" 2>&3
 kind: ConfigMap
 metadata:
   name: pg-backup-script

templates/cm_postgres_bkp.yaml

@@ -18,4 +18,4 @@ data:
   WAL_S3_BUCKET: postgres
   WAL_BUCKET_SCOPE_PREFIX: ""
   WAL_BUCKET_SCOPE_SUFFIX: ""
-  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']"
+  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','30 2 * * * /scripts/backup_retry.sh']"

values_cusqa.yaml

@@ -90,6 +90,16 @@ smardigo-connect:
               defaultMode: 0777
           targetContainers:
             - postgres
+        -
+          name: pg-reschedulebackup-script
+          mountPath: /scripts/backup_retry.sh
+          subPath: backup_retry.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              defaultMode: 0777
+          targetContainers:
+            - postgres
     monitoring:
       alerts:
         postgres:

values_nsodev.yaml

@@ -90,6 +90,16 @@ smardigo-connect:
               defaultMode: 0777
           targetContainers:
             - postgres
+        -
+          name: pg-reschedulebackup-script
+          mountPath: /scripts/backup_retry.sh
+          subPath: backup_retry.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              defaultMode: 0777
+          targetContainers:
+            - postgres
     monitoring:
       alerts:
         postgres: