diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml new file mode 100644 index 0000000..32d3382 --- /dev/null +++ b/templates/cm_pg_backup_scripts.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +data: + nsodevops_postgres_backup_wrapper.sh: | + #!/bin/bash + + + + envdir "/run/etc/wal-e.d/env" bash -x /nsoscripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" >> /tmp/backup_cron_`date +%F`.log + postgres_backup.sh: | + #!/bin/bash + + # fgoerz + + function log + { + echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*" + } + + [[ -z $1 ]] && echo "Usage: $0 PGDATA" && exit 1 + + log "I was called as: $0 $*" + + + readonly PGDATA=$1 + DAYS_TO_RETAIN=$BACKUP_NUM_TO_RETAIN + + readonly IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()") + if [[ $IN_RECOVERY == "f" ]]; then + [[ "$WALG_BACKUP_FROM_REPLICA" == "true" ]] && log "Cluster is not in recovery, not running backup" && exit 0 + elif [[ $IN_RECOVERY == "t" ]]; then + [[ "$WALG_BACKUP_FROM_REPLICA" != "true" ]] && log "Cluster is in recovery, not running backup" && exit 0 + else + log "ERROR: Recovery state unknown: $IN_RECOVERY" && exit 1 + fi + + # leave at least 2 days base backups before creating a new one + [[ "$DAYS_TO_RETAIN" -lt 2 ]] && DAYS_TO_RETAIN=2 + + if [[ "$USE_WALG_BACKUP" == "true" ]]; then + readonly WAL_E="wal-g" + [[ -z $WALG_BACKUP_COMPRESSION_METHOD ]] || export WALG_COMPRESSION_METHOD=$WALG_BACKUP_COMPRESSION_METHOD + export PGHOST=/var/run/postgresql + else + readonly WAL_E="wal-e" + + # Ensure we don't have more workes than CPU's + POOL_SIZE=$(grep -c ^processor /proc/cpuinfo 2>/dev/null || 1) + [ "$POOL_SIZE" -gt 4 ] && POOL_SIZE=4 + POOL_SIZE=(--pool-size "$POOL_SIZE") + fi + + BEFORE="" + LEFT=0 + + readonly NOW=$(date +%s -u) + while read -r name last_modified rest; do + last_modified=$(date +%s -ud "$last_modified") + if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then + if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then + BEFORE_TIME=$last_modified + BEFORE=$name + fi + else + # count how many backups will remain after we remove everything up to certain date + ((LEFT=LEFT+1)) + fi + done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') + + # we want keep at least N backups even if the number of days exceeded + if [ ! -z "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then + if [[ "$USE_WALG_BACKUP" == "true" ]]; then + $WAL_E delete before FIND_FULL "$BEFORE" --confirm + else + $WAL_E delete --confirm before "$BEFORE" + fi + fi + + # push a new base backup + log "producing a new backup" + # We reduce the priority of the backup for CPU consumption + exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" +kind: ConfigMap +metadata: + name: pg-backup-script diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml index d59b35d..fb33881 100644 --- a/templates/cm_postgres_bkp.yaml +++ b/templates/cm_postgres_bkp.yaml @@ -10,7 +10,7 @@ data: AWS_REGION: "" AWS_S3_FORCE_PATH_STYLE: "true" # needed for MinIO BACKUP_NUM_TO_RETAIN: "7" - BACKUP_SCHEDULE: "00 2 * * *" +# BACKUP_SCHEDULE: "00 2 * * *" CLONE_USE_WALG_RESTORE: "true" USE_WALG_BACKUP: "true" USE_WALG_RESTORE: "true" @@ -18,4 +18,4 @@ data: WAL_S3_BUCKET: postgres WAL_BUCKET_SCOPE_PREFIX: "" WAL_BUCKET_SCOPE_SUFFIX: "" - CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']" + CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','00 2 * * * /nsoscripts/nsodevops_postgres_backup_wrapper.sh']" diff --git a/values_nsodev.yaml b/values_nsodev.yaml index 3bf404d..c5d6a1e 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -70,6 +70,43 @@ smardigo-connect: spec: volume: size: 11Gi + additionalVolumes: + - + name: backup-monitoring-script + mountPath: /nso_scripts/backup-monitoring.sh + volumeSource: + configMap: + name: backup-monitoring-script + items: + - key: backup-monitoring.sh + path: backup-monitoring.sh + defaultMode: 0777 + targetContainers: + - postgres + - + name: pg-backup-script + mountPath: /nsoscripts/postgres_backup.sh + volumeSource: + configMap: + name: pg-backup-script + items: + - key: postgres_backup.sh + path: postgres_backup.sh + defaultMode: 0777 + targetContainers: + - postgres + - + name: pg-backup-wrapper-script + mountPath: /nsoscripts/nsodevops_postgres_backup_wrapper.sh + volumeSource: + configMap: + name: pg-backup-script + items: + - key: nsodevops_postgres_backup_wrapper.sh + path: nsodevops_postgres_backup_wrapper.sh + defaultMode: 0777 + targetContainers: + - postgres monitoring: alerts: postgres: