From acc2a4c759cf1427b58d44108c9fa52bf34fc363 Mon Sep 17 00:00:00 2001
From: friedrich goerz <friedrich.goerz@netgo.de>
Date: Wed, 10 May 2023 14:20:04 +0200
Subject: [PATCH 1/2] added script to retrigger backup in case of backup run
 got broken.  pimped debugging infos in case of errors.

---
 templates/cm_pg_backup_scripts.yaml | 31 ++++++++++++++++++++++++++++-
 templates/cm_postgres_bkp.yaml      |  2 +-
 values_cusqa.yaml                   | 10 ++++++++++
 values_nsodev.yaml                  | 10 ++++++++++
 4 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml
index 6354fdb..e93959e 100644
--- a/templates/cm_pg_backup_scripts.yaml
+++ b/templates/cm_pg_backup_scripts.yaml
@@ -1,11 +1,40 @@
 apiVersion: v1
 data:
+  backup_retry.sh: |
+    #!/bin/bash
+    #
+    #
+    
+    BASENAME=$(basename $0)
+
+    function log
+    {
+        touch /tmp/${BASENAME}.log
+        echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*" >> /tmp/${BASENAME}.log
+    }
+
+    log "INFO - script was executed"
+  
+    LAST_BKP=$(envdir "/run/etc/wal-e.d/env" wal-g backup-list --detail --json | jq -r .[-1].finish_time)
+    
+    LAST_BKP_DATE_IN_UNIXSEC=$(date -d ${LAST_BKP} +"%s")
+    
+    NOW=$(date +%s -u)
+    
+    if [ $((NOW-LAST_BKP_DATE_IN_UNIXSEC)) -lt 86400 ]; then
+      log "INFO - last backup created within 24h. no backup rescheduling needed"
+      exit 0
+    else
+      log "INFO - last backup created older than 24h. triggering backup..."
+      envdir "/run/etc/wal-e.d/env" /scripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data"
+    fi
   postgres_backup.sh: |
     #!/bin/bash
 
     echo "Plz check DEBUG_LOG for debugging purpose. EVERY output will be redirected!"
     # fgoerz DEV-1029
     # pipe all output to file for debugging purpose
+    exec 3>&1
     exec 1>>/tmp/pg_backup_`date +%F`.log
     exec 2>&1
 
@@ -76,7 +105,7 @@ data:
     # push a new base backup
     log "producing a new backup"
     # We reduce the priority of the backup for CPU consumption
-    exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}"
+    exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" 2>&3
 kind: ConfigMap
 metadata:
   name: pg-backup-script
diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml
index d59b35d..d91b943 100644
--- a/templates/cm_postgres_bkp.yaml
+++ b/templates/cm_postgres_bkp.yaml
@@ -18,4 +18,4 @@ data:
   WAL_S3_BUCKET: postgres
   WAL_BUCKET_SCOPE_PREFIX: ""
   WAL_BUCKET_SCOPE_SUFFIX: ""
-  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']"
+  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','30 2 * * * /scripts/backup_retry.sh']"
\ No newline at end of file
diff --git a/values_cusqa.yaml b/values_cusqa.yaml
index 45b1381..1d06491 100644
--- a/values_cusqa.yaml
+++ b/values_cusqa.yaml
@@ -90,6 +90,16 @@ smardigo-connect:
               defaultMode: 0777
           targetContainers:
             - postgres
+        -
+          name: pg-backup-script
+          mountPath: /scripts/backup_retry.sh
+          subPath: backup_retry.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              defaultMode: 0777
+          targetContainers:
+            - postgres
     monitoring:
       alerts:
         postgres:
diff --git a/values_nsodev.yaml b/values_nsodev.yaml
index 72af017..c25bdb2 100644
--- a/values_nsodev.yaml
+++ b/values_nsodev.yaml
@@ -90,6 +90,16 @@ smardigo-connect:
               defaultMode: 0777
           targetContainers:
             - postgres
+        -
+          name: pg-backup-script
+          mountPath: /scripts/backup_retry.sh
+          subPath: backup_retry.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              defaultMode: 0777
+          targetContainers:
+            - postgres
     monitoring:
       alerts:
         postgres:

From 83347cbfbdea2a222b3032de5ea3a19149a7f101 Mon Sep 17 00:00:00 2001
From: friedrich goerz <friedrich.goerz@netgo.de>
Date: Wed, 10 May 2023 15:10:27 +0200
Subject: [PATCH 2/2] DEV-1029: bugfixing silly config error

---
 values_cusqa.yaml  | 2 +-
 values_nsodev.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/values_cusqa.yaml b/values_cusqa.yaml
index 1d06491..f653dab 100644
--- a/values_cusqa.yaml
+++ b/values_cusqa.yaml
@@ -91,7 +91,7 @@ smardigo-connect:
           targetContainers:
             - postgres
         -
-          name: pg-backup-script
+          name: pg-reschedulebackup-script
           mountPath: /scripts/backup_retry.sh
           subPath: backup_retry.sh
           volumeSource:
diff --git a/values_nsodev.yaml b/values_nsodev.yaml
index c25bdb2..88c39b8 100644
--- a/values_nsodev.yaml
+++ b/values_nsodev.yaml
@@ -91,7 +91,7 @@ smardigo-connect:
           targetContainers:
             - postgres
         -
-          name: pg-backup-script
+          name: pg-reschedulebackup-script
           mountPath: /scripts/backup_retry.sh
           subPath: backup_retry.sh
           volumeSource: