DEV-1029: added debugging stuff to hopefully get more insights

templates/cm_pg_backup_scripts.yaml

@@ -0,0 +1,84 @@
+apiVersion: v1
+data:
+  nsodevops_postgres_backup_wrapper.sh: |
+    #!/bin/bash
+
+
+
+    envdir "/run/etc/wal-e.d/env" bash -x /nsoscripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" >> /tmp/backup_cron_`date +%F`.log
+  postgres_backup.sh: |
+    #!/bin/bash
+
+    # fgoerz
+
+    function log
+    {
+        echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*"
+    }
+
+    [[ -z $1 ]] && echo "Usage: $0 PGDATA" && exit 1
+
+    log "I was called as: $0 $*"
+
+
+    readonly PGDATA=$1
+    DAYS_TO_RETAIN=$BACKUP_NUM_TO_RETAIN
+
+    readonly IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()")
+    if [[ $IN_RECOVERY == "f" ]]; then
+        [[ "$WALG_BACKUP_FROM_REPLICA" == "true" ]] && log "Cluster is not in recovery, not running backup" && exit 0
+    elif [[ $IN_RECOVERY == "t" ]]; then
+        [[ "$WALG_BACKUP_FROM_REPLICA" != "true" ]] && log "Cluster is in recovery, not running backup" && exit 0
+    else
+        log "ERROR: Recovery state unknown: $IN_RECOVERY" && exit 1
+    fi
+
+    # leave at least 2 days base backups before creating a new one
+    [[ "$DAYS_TO_RETAIN" -lt 2 ]] && DAYS_TO_RETAIN=2
+
+    if [[ "$USE_WALG_BACKUP" == "true" ]]; then
+        readonly WAL_E="wal-g"
+        [[ -z $WALG_BACKUP_COMPRESSION_METHOD ]] || export WALG_COMPRESSION_METHOD=$WALG_BACKUP_COMPRESSION_METHOD
+        export PGHOST=/var/run/postgresql
+    else
+        readonly WAL_E="wal-e"
+
+        # Ensure we don't have more workes than CPU's
+        POOL_SIZE=$(grep -c ^processor /proc/cpuinfo 2>/dev/null || 1)
+        [ "$POOL_SIZE" -gt 4 ] && POOL_SIZE=4
+        POOL_SIZE=(--pool-size "$POOL_SIZE")
+    fi
+
+    BEFORE=""
+    LEFT=0
+
+    readonly NOW=$(date +%s -u)
+    while read -r name last_modified rest; do
+        last_modified=$(date +%s -ud "$last_modified")
+        if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
+            if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then
+                BEFORE_TIME=$last_modified
+                BEFORE=$name
+            fi
+        else
+            # count how many backups will remain after we remove everything up to certain date
+            ((LEFT=LEFT+1))
+        fi
+    done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
+
+    # we want keep at least N backups even if the number of days exceeded
+    if [ ! -z "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
+        if [[ "$USE_WALG_BACKUP" == "true" ]]; then
+            $WAL_E delete before FIND_FULL "$BEFORE" --confirm
+        else
+            $WAL_E delete --confirm before "$BEFORE"
+        fi
+    fi
+
+    # push a new base backup
+    log "producing a new backup"
+    # We reduce the priority of the backup for CPU consumption
+    exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}"
+kind: ConfigMap
+metadata:
+  name: pg-backup-script

templates/cm_postgres_bkp.yaml

@@ -10,7 +10,7 @@ data:
   AWS_REGION: ""
   AWS_S3_FORCE_PATH_STYLE: "true" # needed for MinIO
   BACKUP_NUM_TO_RETAIN: "7"
-  BACKUP_SCHEDULE: "00 2 * * *"
+#  BACKUP_SCHEDULE: "00 2 * * *"
   CLONE_USE_WALG_RESTORE: "true"
   USE_WALG_BACKUP: "true"
   USE_WALG_RESTORE: "true"
@@ -18,4 +18,4 @@ data:
   WAL_S3_BUCKET: postgres
   WAL_BUCKET_SCOPE_PREFIX: ""
   WAL_BUCKET_SCOPE_SUFFIX: ""
-  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']"
+  CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','00 2 * * * /nsoscripts/nsodevops_postgres_backup_wrapper.sh']"

values_nsodev.yaml

@@ -70,6 +70,43 @@ smardigo-connect:
     spec:
       volume:
         size: 11Gi
+      additionalVolumes:
+        -
+          name: backup-monitoring-script
+          mountPath: /nso_scripts/backup-monitoring.sh
+          volumeSource:
+            configMap:
+              name: backup-monitoring-script
+              items:
+              - key: backup-monitoring.sh
+                path: backup-monitoring.sh
+              defaultMode: 0777
+          targetContainers:
+            - postgres
+        -
+          name: pg-backup-script
+          mountPath: /nsoscripts/postgres_backup.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              items:
+              - key: postgres_backup.sh
+                path: postgres_backup.sh
+              defaultMode: 0777
+          targetContainers:
+            - postgres
+        -
+          name: pg-backup-wrapper-script
+          mountPath: /nsoscripts/nsodevops_postgres_backup_wrapper.sh
+          volumeSource:
+            configMap:
+              name: pg-backup-script
+              items:
+              - key: nsodevops_postgres_backup_wrapper.sh
+                path: nsodevops_postgres_backup_wrapper.sh
+              defaultMode: 0777
+          targetContainers:
+            - postgres
     monitoring:
       alerts:
         postgres: