From 959f023fff72990d71dfb5d62f8e14beb5efc662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20H=C3=A4hnel?= Date: Tue, 21 Mar 2023 13:33:13 +0000 Subject: [PATCH 01/16] Nsodev --- Chart.yaml | 2 +- values_cusprod.yaml | 2 ++ values_cusqa.yaml | 2 ++ values_nsodev.yaml | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Chart.yaml b/Chart.yaml index 5751cc9..1401bff 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -45,6 +45,6 @@ dependencies: condition: egeld24-wordpress.enabled alias: egeld24-wordpress - name: iam - version: 0.1.2 + version: 0.1.4 repository: oci://prodnso-harbor-01.smardigo.digital/infrastructure condition: iam.enabled diff --git a/values_cusprod.yaml b/values_cusprod.yaml index 16d2fb4..76f6158 100644 --- a/values_cusprod.yaml +++ b/values_cusprod.yaml @@ -188,3 +188,5 @@ iam: secret_name_keycloak_creds: iam-keycloak-creds envvars: iam_keycloak_auth_server_url: "https://keycloak-prodwork01.smardigo.digital/auth/" + netpols: + keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital \ No newline at end of file diff --git a/values_cusqa.yaml b/values_cusqa.yaml index b14f650..fb25f3b 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -175,3 +175,5 @@ iam: secret_name_keycloak_creds: iam-keycloak-creds envvars: iam_keycloak_auth_server_url: "https://keycloak-prodwork01.smardigo.digital/auth/" + netpols: + keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital \ No newline at end of file diff --git a/values_nsodev.yaml b/values_nsodev.yaml index 7aef5b0..cbb65a0 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -158,3 +158,5 @@ iam: secret_name_keycloak_creds: iam-keycloak-creds envvars: iam_keycloak_auth_server_url: "https://keycloak-prodwork01.smardigo.digital/auth/" + netpols: + keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital From afbaf2f4d5ca25a39a85fb7d42c608ba3ea1ac11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20H=C3=A4hnel?= Date: Mon, 27 Mar 2023 19:37:47 +0000 Subject: [PATCH 02/16] Nsodev --- Chart.yaml | 2 +- values_cusprod.yaml | 6 ++++-- values_cusqa.yaml | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 1401bff..c1cd10e 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -31,7 +31,7 @@ dependencies: version: 4.2.1 repository: oci://prodnso-harbor-01.smardigo.digital/infrastructure - name: smardigo-connect - version: 0.1.27 + version: 0.1.32 repository: oci://prodnso-harbor-01.smardigo.digital/smardigo - name: smardigo-worker version: 0.1.13 diff --git a/values_cusprod.yaml b/values_cusprod.yaml index 76f6158..98efc3c 100644 --- a/values_cusprod.yaml +++ b/values_cusprod.yaml @@ -93,6 +93,8 @@ smardigo-connect: additional_labels: restart: 20230131-1854 domain: connect-cusprod-prodwork01.smardigo.digital + ingress: + customIpWhitelist: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" image: version: 10.4.10 oidc: @@ -140,8 +142,8 @@ smardigo-wordpress: - "*.connect-wordpress-cusprod-prodwork01.smardigo.digital" secretName: "connect-wordpress-cusprod-prodwork01.smardigo.digital-wildcard-tls" annotations: - nginx.ingress.kubernetes.io/whitelist-source-range: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" - customIpWhitelist: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" + nginx.ingress.kubernetes.io/whitelist-source-range: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" + customIpWhitelist: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" commonLabels: app: wordpress project: thgquotenservice diff --git a/values_cusqa.yaml b/values_cusqa.yaml index fb25f3b..47a1aa7 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -80,6 +80,8 @@ smardigo-connect: additional_labels: restart: 20230217-1145 domain: connect-cusqa-prodwork01.smardigo.digital + ingress: + customIpWhitelist: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" image: version: 10.4.13 oidc: From 47344c09fa327fe62f0dbd7a15793dae3dec9867 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 29 Mar 2023 15:51:31 +0200 Subject: [PATCH 03/16] NOTICKET: inc.timesthreshold for wordpress backup alert due to nighlty false positive alerts (cherry picked from commit 9c6aec87babdd6ab8437d5f33388952a461f615e) --- secrets_cusprod.yaml | 5 +++-- secrets_cusqa.yaml | 5 +++-- secrets_nsodev.yaml | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/secrets_cusprod.yaml b/secrets_cusprod.yaml index a826882..c0106ee 100644 --- a/secrets_cusprod.yaml +++ b/secrets_cusprod.yaml @@ -66,6 +66,7 @@ smardigo-wordpress: s3BucketPath: ENC[AES256_GCM,data:yeF/XUDHtiLNb17TNHQ=,iv:srckG428aQkE+exjOz8QREdF2lYWlx8LDDEz9tbNLHo=,tag:TzgkQJrWaATCRzN7wGC/1A==,type:str] prometheusPushgatewayUrl: ENC[AES256_GCM,data:aFlr5l+pAl6i0QFRcIbil/6vw3qy+ucx66h+HwNCBSPjMEgcaWtTbSfaZq2QORM6JiTDw9InOkUok2x90B+iziPzrteiVBq8CdEqOHTY,iv:ECQ9lKTdYfa+dYW5pkMS5MMICcHjQdM+m1K1Hqs1rMs=,tag:WoUWWiQFQGiYVb36xAj0/w==,type:str] stage: ENC[AES256_GCM,data:f/SUEVYTUBONIDmj2Sw=,iv:qLGoVjQvRdOSK/TQ1FgCyKyIuLVdpQeR5kfAboDreeQ=,tag:t2GL2LZ52xEgYsUC6nf/NQ==,type:str] + timeThreshold: ENC[AES256_GCM,data:nUIZwMYh,iv:FO2QwCMkLGZtUF4C6govZF8ZIy/YNRbG+R0yJXZX9fk=,tag:zcxk2kpnVm8QaFHcBT0JUQ==,type:int] wordpress: wordpressUsername: ENC[AES256_GCM,data:6Mp1vUWw9jhyLgxJf0AF,iv:LWeoHzRFs1goHymecK2Zh6VClFqZARh8i0mFk4Wv9Iw=,tag:lMm20O1qwjOLY1WEfVI+0A==,type:str] wordpressPassword: ENC[AES256_GCM,data:eJQlxihckFrScSS+HE1Q4RfuuRgBtSxC,iv:bNKf5VKXbWStyHOUJXAhDR2kXOsO1Nta8QsYY+K+Xkw=,tag:s0pD8OBZO5Ax/5ksXK0QDA==,type:str] @@ -171,8 +172,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2023-03-16T12:06:19Z" - mac: ENC[AES256_GCM,data:iDZ42cVx8i7nWO1Q0Py9/j6cK4i+k6TpxvZp7PV2zPKv2tpayfFtYRn+5TJ1BzXbsy2NKpVzaOk/N3BSmUkhqHiJa4Nr5RHUUTRHRZdskf4hVdCKUImb3fZUctoJkWnkjyqS++ZydMmVxFVKJd+5X2DrmwmeOLiLV0bWK6yHnMA=,iv:RzWVkxe1WrUPHQO0/LRiEQUY81s42NdcCoMnWoxzY/w=,tag:xnrYhFDhMK81LmsiXjFm/g==,type:str] + lastmodified: "2023-03-29T08:39:52Z" + mac: ENC[AES256_GCM,data:1tQmaqeNiGPk27+eNeTS3G5PJvlXKyrSWsfqQzkKbmIEd8nVPQVPxZ7cASP700PBZFe20MdGuv5gq9j6MKd5hEG6kex5Q5V6IgC1iWEn71AzcWFvpwHiVEifvjJjFYXa2Bxg4a3RYu1qIReprSdOzg9raDQu2iNY7patAPcOZNw=,iv:lEvReF2XmGWm+r1MJFi7FFpLMkXTHLWweV0nfNewCnQ=,tag:WSdTfxPxT7jPGMFLWHKf2w==,type:str] pgp: - created_at: "2023-03-10T14:04:21Z" enc: | diff --git a/secrets_cusqa.yaml b/secrets_cusqa.yaml index 11f6679..0ef044e 100644 --- a/secrets_cusqa.yaml +++ b/secrets_cusqa.yaml @@ -66,6 +66,7 @@ smardigo-wordpress: s3BucketPath: ENC[AES256_GCM,data:2gn8KY+HD0isdWlJ91zgmp7KhpPze8MSEyGCu6yWnF61rT8Gtg==,iv:6xTwcitmEJ17yHYjAtfMOpyWpTLmHD4qWkKtIkQQohg=,tag:xbC/srm6goaw3zh6TTW/wQ==,type:str] prometheusPushgatewayUrl: ENC[AES256_GCM,data:V6Kte845PNqWuiJuOCBsG/WawveEWeuOjgbVITbv+LxwySHhzQKQVE4OryDVcNkHKfrz1yTM3/+1bHoNyOtpmnhKrL/c/pMNozytfeth,iv:LD8q1EYtinzmDijQgkY/LwVaAWQJAYW0w4NBayLgHoU=,tag:3NaoQ3wKoMGrnIfcdTCVxA==,type:str] stage: ENC[AES256_GCM,data:0NtdZVRmuO6VUjrs,iv:OAKdor9v/G0vTspRJOgJ2HmBWXCrYiqPgnnLafTQ6B0=,tag:cNCa4g8VMj6qHdTsNaG81g==,type:str] + timeThreshold: ENC[AES256_GCM,data:T51geX3s,iv:SEL1IGoNG3QjqzJgKgEZSgONGX2tDFMDYOR+MXWNypk=,tag:mwHNur3+ZWZWARMBuBm/gQ==,type:int] wordpress: wordpressUsername: ENC[AES256_GCM,data:DoCjbxBz/RWGp5HUViV7,iv:JCjl246lKeIHDGbuDImPYY+WcHauSJi01tAlzLCEGfg=,tag:PnddeyKG77AufJZ9/9ggqQ==,type:str] wordpressPassword: ENC[AES256_GCM,data:sY7ypqlLaDdLS9fRYzGQFVfEMf3h5xqv,iv:7hVX2Be5ctHC6YLZ8OD/TDM97LRFgffj9WLL3yzI86s=,tag:P5/ZK9JLnAzpZVIsWzyVMQ==,type:str] @@ -171,8 +172,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2023-03-16T12:06:08Z" - mac: ENC[AES256_GCM,data:trwXyPe/7aGPzSaNUFAPO/9N1sBoiEf/XwlqY15fjAXCyisgF6PyW8Mu20oUSmb3o7RHO5eGd0tjXNqsiaANc6c7VSlYle4mPq+IE0K5ECNv42FnrSqoin5QlwqgWdjU/N8owzXhSOXs3idE757BLxj8vYPelnEUZM4SUKpaaS0=,iv:irszuCfmCI+e942NvJl2FWyAos6TSHsgKCs+p9ofrR4=,tag:8kLkA2EVIRAU1gqfHwSl6g==,type:str] + lastmodified: "2023-03-29T08:39:44Z" + mac: ENC[AES256_GCM,data:zJ4Czqasn1238HK3lkWbJ+yiZAIcV5rWt4bShvRBfGsfraGKD6k+j+cy6uMb4zkmQREEAJDiBDeyrxtWpz6sKQDZ/VHao252NsYRtKNEvMdo1jVTB0JiwPV4zG8isfn42jyxTfGdrG93nb6Vt3/57c0AqFyOAb0moGBBwRSTC4U=,iv:PjWE1RNNDEupQz6vWsOCnzhGkMh+dfeG5a08aure+jw=,tag:FOGzx4/ZiU3yrsAghtA5jA==,type:str] pgp: - created_at: "2023-03-10T14:04:27Z" enc: | diff --git a/secrets_nsodev.yaml b/secrets_nsodev.yaml index 6ce68cc..f61d773 100644 --- a/secrets_nsodev.yaml +++ b/secrets_nsodev.yaml @@ -66,6 +66,7 @@ smardigo-wordpress: s3BucketPath: ENC[AES256_GCM,data:VvJq4A3H6CSw8DAfN87T,iv:dexgBg0Ac4kgFZM/kBu5tvyAhl2VNaQ5zi90GwKZ8CU=,tag:3QE90dY+4UmtIY7fQq8Ciw==,type:str] prometheusPushgatewayUrl: ENC[AES256_GCM,data:8yghvPPAIKze8886t62ppuynhC/yb/zINp5Cnw/iAkB51A73m9lC+5b7pdiZLf+d+RZqXGKrjkqxUFE56QUJvKQgHUkIFOeBwagW3Ix6,iv:oTns/pcHpYuCwxQavHjTjlL0vI0PBQIQmISY7YleM4c=,tag:49ykjY/UVX+ypzxa6J7lrg==,type:str] stage: ENC[AES256_GCM,data:waGh8ukUaW2m9skaHg==,iv:pA2NjGhWSH+ySVi5e4x4GbBZ37neUx9YWOZo1uqflFY=,tag:5PjWMouMJszbdRs/wEaIIQ==,type:str] + timeThreshold: ENC[AES256_GCM,data:14v0rlqa,iv:/sxHrAEoY5B4VIjufb2LgLkvMknOqPnEP904/KxZLoU=,tag:4xln6bQKVzupNaKa9s9sDA==,type:int] wordpress: wordpressUsername: ENC[AES256_GCM,data:fw8FcJ9SlVNvTe2mpgTB,iv:XnvZHV09Q9f+uujQzAUvZPFDuqJwmIgfi4T78b0FIwc=,tag:iZX9BOWGeZZQew7RUuR8bw==,type:str] wordpressPassword: ENC[AES256_GCM,data:eLyWLWEt8eCihFrrwMOvq+CmzmAAIVTV,iv:vA2qcQJ8nfEiGBQpYxiIPVTUFMGbwj3DRVJqoe8YSlg=,tag:76vOI+9ahNBn4s67XjsjIQ==,type:str] @@ -120,8 +121,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2023-03-16T09:21:59Z" - mac: ENC[AES256_GCM,data:6IfgCi6LspJo1ui0x3G3lgOJojTEM6pznHsFaKz6cUf4D4B0S5AWspawgWB9gD+Lq1AHI5zfF0Q2USqF+QjhqK7YQ5nwDTtN4h3G6DN4gdvnxxM1EJUt9WrVTFYaQLcY/lYWaZ/0YQH4/LBbAilPZt6GM0fyE+e6bolhwq8jYcQ=,iv:xJTZFyFy3g1ctnn3i97cU3OOktS7romdsze9xV+FoyY=,tag:7D/qgLjzM+BAM1GQny3C0w==,type:str] + lastmodified: "2023-03-29T08:39:34Z" + mac: ENC[AES256_GCM,data:hCfc0j96XYhbpdPp+gIpSH/sgrOP91dfgLh68v2bBrU3YVpIzji9dQbNX7vonqj82KpLDaD92isD/tpP0eIho4wHO5tfOvYZ2nyoLvMGnLWUy7EtALJ60bgG7aPArSfXF5I4wRJLXl018yeawaeFJ/qlApWv+v02CDSj523VSiY=,iv:GCos9Qdxpu8BsPTgY1quB4bCwESwkZ9OP6RooPaaAII=,tag:vavzSflJw3LufUt6LgNH7Q==,type:str] pgp: - created_at: "2023-02-17T10:43:42Z" enc: | From 3dc950aee1ba048bc929e8481037f4d1e9447a6b Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Mon, 17 Apr 2023 14:30:56 +0200 Subject: [PATCH 04/16] DEV-866: switching to higher version of dep.helm chart due to removing double declaraion of database in kind postgresql (cherry picked from commit f83385f8e58dea15135fff63cf8069d1747aa52d) --- Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Chart.yaml b/Chart.yaml index c1cd10e..b68aee2 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -31,7 +31,7 @@ dependencies: version: 4.2.1 repository: oci://prodnso-harbor-01.smardigo.digital/infrastructure - name: smardigo-connect - version: 0.1.32 + version: 0.1.33 repository: oci://prodnso-harbor-01.smardigo.digital/smardigo - name: smardigo-worker version: 0.1.13 From 6e0491781726c3a2f494185e9353f6485b5dde1b Mon Sep 17 00:00:00 2001 From: Johannes Wicovsky Date: Wed, 26 Apr 2023 07:59:12 +0200 Subject: [PATCH 05/16] Merge branch 'nsodev' into cusqa --- Chart.yaml | 6 +++--- values_cusqa.yaml | 4 +++- values_nsodev.yaml | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index b68aee2..603c011 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.38 +version: 0.1.39 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -37,10 +37,10 @@ dependencies: version: 0.1.13 repository: oci://prodnso-harbor-01.smardigo.digital/smardigo - name: smardigo-wordpress - version: 0.1.34 + version: 0.1.38 repository: oci://prodnso-harbor-01.smardigo.digital/smardigo - name: smardigo-wordpress - version: 0.1.34 + version: 0.1.38 repository: oci://prodnso-harbor-01.smardigo.digital/smardigo condition: egeld24-wordpress.enabled alias: egeld24-wordpress diff --git a/values_cusqa.yaml b/values_cusqa.yaml index 47a1aa7..3d0552b 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -103,12 +103,14 @@ smardigo-connect: smardigo-worker: uba: enabled: true + image: + version: 10.5.0 sepa: enabled: true ocr: enabled: true image: - version: 10.4.5 + version: 10.4.6 smardigo-wordpress: wordpress: diff --git a/values_nsodev.yaml b/values_nsodev.yaml index cbb65a0..3bf404d 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -101,12 +101,14 @@ smardigo-connect: smardigo-worker: uba: enabled: true + image: + version: 10.5.0 sepa: enabled: true ocr: enabled: true image: - version: 10.4.5 + version: 10.4.6 smardigo-wordpress: wordpress: From e4b93d9ee8316db58b07a014aef0477b6cfbefa1 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 13:04:31 +0200 Subject: [PATCH 06/16] DEV-1029: added debugging stuff to hopefully get more insights --- templates/cm_pg_backup_scripts.yaml | 84 +++++++++++++++++++++++++++++ templates/cm_postgres_bkp.yaml | 4 +- values_nsodev.yaml | 37 +++++++++++++ 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 templates/cm_pg_backup_scripts.yaml diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml new file mode 100644 index 0000000..32d3382 --- /dev/null +++ b/templates/cm_pg_backup_scripts.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +data: + nsodevops_postgres_backup_wrapper.sh: | + #!/bin/bash + + + + envdir "/run/etc/wal-e.d/env" bash -x /nsoscripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" >> /tmp/backup_cron_`date +%F`.log + postgres_backup.sh: | + #!/bin/bash + + # fgoerz + + function log + { + echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*" + } + + [[ -z $1 ]] && echo "Usage: $0 PGDATA" && exit 1 + + log "I was called as: $0 $*" + + + readonly PGDATA=$1 + DAYS_TO_RETAIN=$BACKUP_NUM_TO_RETAIN + + readonly IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()") + if [[ $IN_RECOVERY == "f" ]]; then + [[ "$WALG_BACKUP_FROM_REPLICA" == "true" ]] && log "Cluster is not in recovery, not running backup" && exit 0 + elif [[ $IN_RECOVERY == "t" ]]; then + [[ "$WALG_BACKUP_FROM_REPLICA" != "true" ]] && log "Cluster is in recovery, not running backup" && exit 0 + else + log "ERROR: Recovery state unknown: $IN_RECOVERY" && exit 1 + fi + + # leave at least 2 days base backups before creating a new one + [[ "$DAYS_TO_RETAIN" -lt 2 ]] && DAYS_TO_RETAIN=2 + + if [[ "$USE_WALG_BACKUP" == "true" ]]; then + readonly WAL_E="wal-g" + [[ -z $WALG_BACKUP_COMPRESSION_METHOD ]] || export WALG_COMPRESSION_METHOD=$WALG_BACKUP_COMPRESSION_METHOD + export PGHOST=/var/run/postgresql + else + readonly WAL_E="wal-e" + + # Ensure we don't have more workes than CPU's + POOL_SIZE=$(grep -c ^processor /proc/cpuinfo 2>/dev/null || 1) + [ "$POOL_SIZE" -gt 4 ] && POOL_SIZE=4 + POOL_SIZE=(--pool-size "$POOL_SIZE") + fi + + BEFORE="" + LEFT=0 + + readonly NOW=$(date +%s -u) + while read -r name last_modified rest; do + last_modified=$(date +%s -ud "$last_modified") + if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then + if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then + BEFORE_TIME=$last_modified + BEFORE=$name + fi + else + # count how many backups will remain after we remove everything up to certain date + ((LEFT=LEFT+1)) + fi + done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') + + # we want keep at least N backups even if the number of days exceeded + if [ ! -z "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then + if [[ "$USE_WALG_BACKUP" == "true" ]]; then + $WAL_E delete before FIND_FULL "$BEFORE" --confirm + else + $WAL_E delete --confirm before "$BEFORE" + fi + fi + + # push a new base backup + log "producing a new backup" + # We reduce the priority of the backup for CPU consumption + exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" +kind: ConfigMap +metadata: + name: pg-backup-script diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml index d59b35d..fb33881 100644 --- a/templates/cm_postgres_bkp.yaml +++ b/templates/cm_postgres_bkp.yaml @@ -10,7 +10,7 @@ data: AWS_REGION: "" AWS_S3_FORCE_PATH_STYLE: "true" # needed for MinIO BACKUP_NUM_TO_RETAIN: "7" - BACKUP_SCHEDULE: "00 2 * * *" +# BACKUP_SCHEDULE: "00 2 * * *" CLONE_USE_WALG_RESTORE: "true" USE_WALG_BACKUP: "true" USE_WALG_RESTORE: "true" @@ -18,4 +18,4 @@ data: WAL_S3_BUCKET: postgres WAL_BUCKET_SCOPE_PREFIX: "" WAL_BUCKET_SCOPE_SUFFIX: "" - CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']" + CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','00 2 * * * /nsoscripts/nsodevops_postgres_backup_wrapper.sh']" diff --git a/values_nsodev.yaml b/values_nsodev.yaml index 3bf404d..c5d6a1e 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -70,6 +70,43 @@ smardigo-connect: spec: volume: size: 11Gi + additionalVolumes: + - + name: backup-monitoring-script + mountPath: /nso_scripts/backup-monitoring.sh + volumeSource: + configMap: + name: backup-monitoring-script + items: + - key: backup-monitoring.sh + path: backup-monitoring.sh + defaultMode: 0777 + targetContainers: + - postgres + - + name: pg-backup-script + mountPath: /nsoscripts/postgres_backup.sh + volumeSource: + configMap: + name: pg-backup-script + items: + - key: postgres_backup.sh + path: postgres_backup.sh + defaultMode: 0777 + targetContainers: + - postgres + - + name: pg-backup-wrapper-script + mountPath: /nsoscripts/nsodevops_postgres_backup_wrapper.sh + volumeSource: + configMap: + name: pg-backup-script + items: + - key: nsodevops_postgres_backup_wrapper.sh + path: nsodevops_postgres_backup_wrapper.sh + defaultMode: 0777 + targetContainers: + - postgres monitoring: alerts: postgres: From 1c8796ae15d60778db2ad26465382a2f3b677fd2 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 13:07:19 +0200 Subject: [PATCH 07/16] DEV-1029: added debugging stuff to hopefully get more insights --- values_nsodev.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/values_nsodev.yaml b/values_nsodev.yaml index c5d6a1e..944b46f 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -73,13 +73,10 @@ smardigo-connect: additionalVolumes: - name: backup-monitoring-script - mountPath: /nso_scripts/backup-monitoring.sh + mountPath: /nso_scripts volumeSource: configMap: name: backup-monitoring-script - items: - - key: backup-monitoring.sh - path: backup-monitoring.sh defaultMode: 0777 targetContainers: - postgres From fc0366629f911676148830b45d47f7835664f017 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 13:27:35 +0200 Subject: [PATCH 08/16] DEV-1029: added debugging stuff to hopefully get more insights - bugfix --- values_nsodev.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/values_nsodev.yaml b/values_nsodev.yaml index 944b46f..f68dbf9 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -86,21 +86,6 @@ smardigo-connect: volumeSource: configMap: name: pg-backup-script - items: - - key: postgres_backup.sh - path: postgres_backup.sh - defaultMode: 0777 - targetContainers: - - postgres - - - name: pg-backup-wrapper-script - mountPath: /nsoscripts/nsodevops_postgres_backup_wrapper.sh - volumeSource: - configMap: - name: pg-backup-script - items: - - key: nsodevops_postgres_backup_wrapper.sh - path: nsodevops_postgres_backup_wrapper.sh defaultMode: 0777 targetContainers: - postgres From 955c708d10f99db82900c06758b94ce59a7dde63 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 13:34:21 +0200 Subject: [PATCH 09/16] DEV-1029: added debugging stuff to hopefully get more insights - bugfix2 --- values_nsodev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values_nsodev.yaml b/values_nsodev.yaml index f68dbf9..f268417 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -82,7 +82,7 @@ smardigo-connect: - postgres - name: pg-backup-script - mountPath: /nsoscripts/postgres_backup.sh + mountPath: /nsoscripts volumeSource: configMap: name: pg-backup-script From e3dfecf4e8aee2135a6c441ea850279aac9a2364 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 22:08:27 +0200 Subject: [PATCH 10/16] DEV-1029: added debugging stuff to hopefully get more insights - bugfix3 --- templates/cm_pg_backup_scripts.yaml | 12 +++++------- templates/cm_postgres_bkp.yaml | 2 +- values_nsodev.yaml | 3 ++- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml index 32d3382..7bfe3a9 100644 --- a/templates/cm_pg_backup_scripts.yaml +++ b/templates/cm_pg_backup_scripts.yaml @@ -1,15 +1,13 @@ apiVersion: v1 data: - nsodevops_postgres_backup_wrapper.sh: | - #!/bin/bash - - - - envdir "/run/etc/wal-e.d/env" bash -x /nsoscripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" >> /tmp/backup_cron_`date +%F`.log postgres_backup.sh: | #!/bin/bash - # fgoerz + DEBUG_LOG="/tmp/pg_backup_`date +%F`.log" + echo "Plz check $DEBUG_LOG for debugging purpose. EVERY output will be redirected!" + # fgoerz DEV-1029 + # pipe all output to file for debugging purpose + exec 2>&1 1>$DEBUG_LOG function log { diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml index fb33881..ae2d9b6 100644 --- a/templates/cm_postgres_bkp.yaml +++ b/templates/cm_postgres_bkp.yaml @@ -10,7 +10,7 @@ data: AWS_REGION: "" AWS_S3_FORCE_PATH_STYLE: "true" # needed for MinIO BACKUP_NUM_TO_RETAIN: "7" -# BACKUP_SCHEDULE: "00 2 * * *" + BACKUP_SCHEDULE: "00 2 * * *" CLONE_USE_WALG_RESTORE: "true" USE_WALG_BACKUP: "true" USE_WALG_RESTORE: "true" diff --git a/values_nsodev.yaml b/values_nsodev.yaml index f268417..72af017 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -82,7 +82,8 @@ smardigo-connect: - postgres - name: pg-backup-script - mountPath: /nsoscripts + mountPath: /scripts/postgres_backup.sh + subPath: postgres_backup.sh volumeSource: configMap: name: pg-backup-script From 1278c452919b6388fc3d367aa0072770811e3606 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 22:09:15 +0200 Subject: [PATCH 11/16] DEV-1029: added debugging stuff to hopefully get more insights - bugfix3 --- templates/cm_postgres_bkp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml index ae2d9b6..d59b35d 100644 --- a/templates/cm_postgres_bkp.yaml +++ b/templates/cm_postgres_bkp.yaml @@ -18,4 +18,4 @@ data: WAL_S3_BUCKET: postgres WAL_BUCKET_SCOPE_PREFIX: "" WAL_BUCKET_SCOPE_SUFFIX: "" - CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','00 2 * * * /nsoscripts/nsodevops_postgres_backup_wrapper.sh']" + CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']" From 3221fa16efa272d94d63d8ba831573e515bd6dbe Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 3 May 2023 22:34:50 +0200 Subject: [PATCH 12/16] DEV-1029: added debugging stuff to hopefully get more insights - bugfix4 --- templates/cm_pg_backup_scripts.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml index 7bfe3a9..6354fdb 100644 --- a/templates/cm_pg_backup_scripts.yaml +++ b/templates/cm_pg_backup_scripts.yaml @@ -3,11 +3,11 @@ data: postgres_backup.sh: | #!/bin/bash - DEBUG_LOG="/tmp/pg_backup_`date +%F`.log" - echo "Plz check $DEBUG_LOG for debugging purpose. EVERY output will be redirected!" + echo "Plz check DEBUG_LOG for debugging purpose. EVERY output will be redirected!" # fgoerz DEV-1029 # pipe all output to file for debugging purpose - exec 2>&1 1>$DEBUG_LOG + exec 1>>/tmp/pg_backup_`date +%F`.log + exec 2>&1 function log { From 7102fd38ecfe91639889499c8018ad883a96daf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6rz=2C=20Friedrich?= Date: Thu, 4 May 2023 10:20:24 +0000 Subject: [PATCH 13/16] Merge branch 'nsodev_DEV-1029_mobenecusqa_pgbkp_broken_bugfixcusqa' into 'nsodev' DEV-1029: added missing pg-cluster params for cusqa See merge request smardigo-hetzner/mobene/prodwork01-mobene-deployment!29 (cherry picked from commit b1c532885e40c0b2f46fc5fa6208f19e696b104c) bbcf7b81 DEV-1029: added missing pg-cluster params for cusqa --- values_cusqa.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/values_cusqa.yaml b/values_cusqa.yaml index 3d0552b..45b1381 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -70,6 +70,26 @@ smardigo-connect: spec: volume: size: 25Gi + additionalVolumes: + - + name: backup-monitoring-script + mountPath: /nso_scripts + volumeSource: + configMap: + name: backup-monitoring-script + defaultMode: 0777 + targetContainers: + - postgres + - + name: pg-backup-script + mountPath: /scripts/postgres_backup.sh + subPath: postgres_backup.sh + volumeSource: + configMap: + name: pg-backup-script + defaultMode: 0777 + targetContainers: + - postgres monitoring: alerts: postgres: @@ -180,4 +200,4 @@ iam: envvars: iam_keycloak_auth_server_url: "https://keycloak-prodwork01.smardigo.digital/auth/" netpols: - keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital \ No newline at end of file + keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital From acc2a4c759cf1427b58d44108c9fa52bf34fc363 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 10 May 2023 14:20:04 +0200 Subject: [PATCH 14/16] added script to retrigger backup in case of backup run got broken. pimped debugging infos in case of errors. --- templates/cm_pg_backup_scripts.yaml | 31 ++++++++++++++++++++++++++++- templates/cm_postgres_bkp.yaml | 2 +- values_cusqa.yaml | 10 ++++++++++ values_nsodev.yaml | 10 ++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/templates/cm_pg_backup_scripts.yaml b/templates/cm_pg_backup_scripts.yaml index 6354fdb..e93959e 100644 --- a/templates/cm_pg_backup_scripts.yaml +++ b/templates/cm_pg_backup_scripts.yaml @@ -1,11 +1,40 @@ apiVersion: v1 data: + backup_retry.sh: | + #!/bin/bash + # + # + + BASENAME=$(basename $0) + + function log + { + touch /tmp/${BASENAME}.log + echo "$(date "+%Y-%m-%d %H:%M:%S.%3N") - $0 - $*" >> /tmp/${BASENAME}.log + } + + log "INFO - script was executed" + + LAST_BKP=$(envdir "/run/etc/wal-e.d/env" wal-g backup-list --detail --json | jq -r .[-1].finish_time) + + LAST_BKP_DATE_IN_UNIXSEC=$(date -d ${LAST_BKP} +"%s") + + NOW=$(date +%s -u) + + if [ $((NOW-LAST_BKP_DATE_IN_UNIXSEC)) -lt 86400 ]; then + log "INFO - last backup created within 24h. no backup rescheduling needed" + exit 0 + else + log "INFO - last backup created older than 24h. triggering backup..." + envdir "/run/etc/wal-e.d/env" /scripts/postgres_backup.sh "/home/postgres/pgdata/pgroot/data" + fi postgres_backup.sh: | #!/bin/bash echo "Plz check DEBUG_LOG for debugging purpose. EVERY output will be redirected!" # fgoerz DEV-1029 # pipe all output to file for debugging purpose + exec 3>&1 exec 1>>/tmp/pg_backup_`date +%F`.log exec 2>&1 @@ -76,7 +105,7 @@ data: # push a new base backup log "producing a new backup" # We reduce the priority of the backup for CPU consumption - exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" + exec nice -n 5 $WAL_E backup-push "$PGDATA" "${POOL_SIZE[@]}" 2>&3 kind: ConfigMap metadata: name: pg-backup-script diff --git a/templates/cm_postgres_bkp.yaml b/templates/cm_postgres_bkp.yaml index d59b35d..d91b943 100644 --- a/templates/cm_postgres_bkp.yaml +++ b/templates/cm_postgres_bkp.yaml @@ -18,4 +18,4 @@ data: WAL_S3_BUCKET: postgres WAL_BUCKET_SCOPE_PREFIX: "" WAL_BUCKET_SCOPE_SUFFIX: "" - CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh']" + CRONTAB: "['* * * * * /nso_scripts/backup-monitoring.sh','30 2 * * * /scripts/backup_retry.sh']" \ No newline at end of file diff --git a/values_cusqa.yaml b/values_cusqa.yaml index 45b1381..1d06491 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -90,6 +90,16 @@ smardigo-connect: defaultMode: 0777 targetContainers: - postgres + - + name: pg-backup-script + mountPath: /scripts/backup_retry.sh + subPath: backup_retry.sh + volumeSource: + configMap: + name: pg-backup-script + defaultMode: 0777 + targetContainers: + - postgres monitoring: alerts: postgres: diff --git a/values_nsodev.yaml b/values_nsodev.yaml index 72af017..c25bdb2 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -90,6 +90,16 @@ smardigo-connect: defaultMode: 0777 targetContainers: - postgres + - + name: pg-backup-script + mountPath: /scripts/backup_retry.sh + subPath: backup_retry.sh + volumeSource: + configMap: + name: pg-backup-script + defaultMode: 0777 + targetContainers: + - postgres monitoring: alerts: postgres: From 83347cbfbdea2a222b3032de5ea3a19149a7f101 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 10 May 2023 15:10:27 +0200 Subject: [PATCH 15/16] DEV-1029: bugfixing silly config error --- values_cusqa.yaml | 2 +- values_nsodev.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/values_cusqa.yaml b/values_cusqa.yaml index 1d06491..f653dab 100644 --- a/values_cusqa.yaml +++ b/values_cusqa.yaml @@ -91,7 +91,7 @@ smardigo-connect: targetContainers: - postgres - - name: pg-backup-script + name: pg-reschedulebackup-script mountPath: /scripts/backup_retry.sh subPath: backup_retry.sh volumeSource: diff --git a/values_nsodev.yaml b/values_nsodev.yaml index c25bdb2..88c39b8 100644 --- a/values_nsodev.yaml +++ b/values_nsodev.yaml @@ -91,7 +91,7 @@ smardigo-connect: targetContainers: - postgres - - name: pg-backup-script + name: pg-reschedulebackup-script mountPath: /scripts/backup_retry.sh subPath: backup_retry.sh volumeSource: From 318a87209525ad1c3737b4e80bd284442a6ae100 Mon Sep 17 00:00:00 2001 From: Michael Haehnel Date: Thu, 1 Jun 2023 17:59:34 +0200 Subject: [PATCH 16/16] DEV-1057 Increase Postgres data storage for cusprod (cherry picked from commit a101d902ad548f3d5fdb6e228931cb3f40e094f5) --- Chart.yaml | 2 +- values_cusprod.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 603c011..e4a4e0a 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.39 +version: 0.1.40 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/values_cusprod.yaml b/values_cusprod.yaml index 98efc3c..752ffce 100644 --- a/values_cusprod.yaml +++ b/values_cusprod.yaml @@ -82,7 +82,7 @@ smardigo-connect: postgres: spec: volume: - size: 100Gi + size: 200Gi monitoring: alerts: postgres: @@ -138,9 +138,9 @@ smardigo-wordpress: - name: "*.connect-wordpress-cusprod-prodwork01.smardigo.digital" path: / extraTls: - - hosts: - - "*.connect-wordpress-cusprod-prodwork01.smardigo.digital" - secretName: "connect-wordpress-cusprod-prodwork01.smardigo.digital-wildcard-tls" + - hosts: + - "*.connect-wordpress-cusprod-prodwork01.smardigo.digital" + secretName: "connect-wordpress-cusprod-prodwork01.smardigo.digital-wildcard-tls" annotations: nginx.ingress.kubernetes.io/whitelist-source-range: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" customIpWhitelist: "212.121.131.106/32,149.233.6.129/32,46.245.219.98/32,164.138.195.162/32,10.0.0.0/16,167.235.150.133/32,167.235.150.197/32,23.88.53.161/32,195.201.113.110/32,5.75.184.216/32,92.42.192.157/32,164.92.251.253/32,91.107.228.133/32,167.235.25.0/32" @@ -191,4 +191,4 @@ iam: envvars: iam_keycloak_auth_server_url: "https://keycloak-prodwork01.smardigo.digital/auth/" netpols: - keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital \ No newline at end of file + keycloak_ip: 167.235.109.35/32 # prodwork01-ingress.smardigo.digital