DEV-895 dedizierter s3 user

create-server.yml

@@ -119,6 +119,7 @@
           'docker-logrotate',
           'docker-engine',
           'smartmontools',
+          'mc',
         ]
         state: 'absent'
       when: ansible_distribution == "Ubuntu"

group_vars/all/plain.yml

@@ -34,7 +34,7 @@ metricbeat_enabled: false
 node_exporter_enabled: true
 
 common_apt_dependencies:
-  - mc
+  - jq
   - vim
   # TODO Check if we really want this
   - zip

group_vars/stage_devnso/backup.yml

@@ -0,0 +1,74 @@
+# Backup space
+#backup_lvm_hcloudvol_size: 30
+#backup_lvm_hcloudvol_count: 8
+
+backup_user_ssh_privkey: "{{ backup_user_ssh_privkey_vault }}"
+
+# Admin access for S3 Storage on stage devnso
+devnso_minio_admin_accesskey: "{{ devnso_minio_admin_accesskey_vault }}"
+devnso_minio_admin_secretkey: "{{ devnso_minio_admin_secretkey_vault }}"
+
+# Readonly access for S3 Storage on stage devnso all buckets
+devnso_minio_user_read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey_vault }}"
+devnso_minio_user_read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage devnso bucket wordpress 
+devnso_minio_user_read_write_wordpress_accesskey: "{{ devnso_minio_user_read_write_wordpress_accesskey_vault }}"
+devnso_minio_user_read_write_wordpress_secretkey: "{{ devnso_minio_user_read_write_wordpress_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage devnso bucket postgres 
+devnso_minio_user_read_write_postgres_accesskey: "{{ devnso_minio_user_read_write_postgres_accesskey_vault }}"
+devnso_minio_user_read_write_postgres_secretkey: "{{ devnso_minio_user_read_write_postgres_secretkey_vault }}"
+
+# Admin access for S3 Storage on "stage" keycloak
+keycloak_minio_admin_accesskey: "{{ keycloak_minio_admin_accesskey_vault }}"
+keycloak_minio_admin_secretkey: "{{ keycloak_minio_admin_secretkey_vault }}"
+
+# Readonly access for S3 Storage on "stage" keycloak all buckets
+keycloak_minio_user_read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey_vault }}"
+keycloak_minio_user_read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on "stage" keycloak bucket postgres 
+keycloak_minio_user_read_write_postgres_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey_vault }}"
+keycloak_minio_user_read_write_postgres_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey_vault }}"
+
+minio_stage_dicts:
+  - {
+      stage: "devnso",
+      url: "https://s3storage-minio-devnso.smardigo.digital",
+      read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ devnso_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ devnso_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ devnso_minio_admin_accesskey }}",
+      admin_secretkey: "{{ devnso_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "30",
+      bucket: "postgres"
+    }
+  - {
+      stage: "devnso",
+      url: "https://s3storage-minio-devnso.smardigo.digital",
+      read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ devnso_minio_user_read_write_wordpress_accesskey }}",
+      read_write_secretkey: "{{ devnso_minio_user_read_write_wordpress_secretkey }}",
+      admin_accesskey: "{{ devnso_minio_admin_accesskey }}",
+      admin_secretkey: "{{ devnso_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "40",
+      bucket: "wordpress"
+    }
+  - {
+      stage: "keycloak",
+      url: "https://s3storage-keycloak-devnso.smardigo.digital",
+      read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ keycloak_minio_admin_accesskey }}",
+      admin_secretkey: "{{ keycloak_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "50",
+      bucket: "postgres"
+    }

group_vars/stage_devnso/vault_backup.yml

@@ -1,28 +1,100 @@
 $ANSIBLE_VAULT;1.1;AES256
-61356237613639353839316532336566616338666264356562386166306466316337393537363930
+63653638613062393562373465663835323662383037386161613336313433663032623733393638
-6438313237336536626636623637643330383637643165630a386334303436613837303435306238
+6334343336396535373865306237613730613032396439630a343238633830336132653832353734
-61323237643831353637653234333932393739393239333466656233383361643038366231373631
+39626662313034346630323864326138313335626565353834333337626164653236333236613566
-3439303035616233640a356431396363383361353561623463656130356130343864363139333765
+6333363364363236310a643666366665333265303435383834613332363334646636636666663366
-33373465383939616662636330343130376235343830643331633531313537663065643439373062
+38663636306232363864356539613933376363333739386634316137306461346231633833343131
-61303631356563383234633430393663386333346231346631643231313462623835373662383238
+62396262316135643031396135316635633766303936623535323031303636666661366431316233
-34353737613030643832373832656339333431343336663131343236353131356466323135356563
+31613066313462366662386463363236666430663430643135383132366536636165653636626633
-38646532303938653535356262653739623333343239346563306161353934633362386239653462
+62633438376361643961373066363836376162316233393038656236346364303530313238333535
-66306363613632356463643132346134396335343966333539373865363863303635316233636632
+32663065393236336131386239366436303731383836376531643262393463393166663931633066
-31656433653839396430656637346465343661376534333936323866376362633432646163326665
+31633732613539333238633364383863393963363261356139396137303563386434353932656365
-34306564613437303634376237356536666435643031313363643265303430336466663865653465
+34653363373665376665616539326635306433383061376332316633623365336366356233666635
-62643865363165386530363632333935623563396364316332636535376330623163306164396530
+63323739326233326538666232623231653839313561656632636539646137313936323138616165
-39366661343136636236313566373732653338326132303031356163646665633534623130383339
+66643162383930633637656134353365626134616533333763306265646133366663306330643363
-63343663313766373664326331663037333631346433333639633936623064613631653234633834
+31336239323638663732383937336263393638616161346132393332353237303534366334613864
-38646437333231353639336534333166313530336662616131663137643965343266633635326335
+63353534313533396438653761623930326262646161386536623333396664343233373135613133
-32353938313639336135386462336436363434636665383339386331336463303035613438383834
+36613033643733616265633832613733613730343537343632336435326432343165316331343439
-61306563373261393833643465633930336465376163353631353166656133323830633632613432
+32656366346665313831353630343831363332336465333364363962326331316363663963373766
-63633830343230396265633831613535613033643538316230373166656566646563306634363761
+62616562646237653263313561616334383437653936613437303735373238333363316539373465
-34326430623838303637343333663537613265636261353765623764666636313330663532306336
+37386636633366663832303133343639303237343963306139613134653232616135393238386635
-66346566653239633965386162666434633662333334386331616235393038636331663438643239
+37363238613962333033376462653562306130366233633035396235333537306234323131373734
-64623965313236366661396663346433363835656633386463376634383537666632663138343931
+32373430323638616166643438313965643630636533663330373139303237656532663437363536
-32613232393265633233343134663630656633303531623764623633373964393431313066613834
+36336634363366653435626166323136323439333063306139633838323034323236626635666564
-35313036666230623263666437306430616139336661383032346132646565616435346166353434
+61306232643337643432373238653935616336613639363964333439623033323339383631313838
-62636466373662303232326437653536326666666639396237613261636633323765393561336464
+39636331376534656337636663323265343839316339616162373332656438363739323366616634
-31613930343934313465616631616663316464303064323861343335653737316463613566636536
+63343966326231396562613038306530303762663338366435373034623439383930373033633730
-36303937643339316164616136393635386436383231393938313534303666636465323666633361
+37373033373065383462343862326338333130323330653633333532316338326363313865316337
-396239303536383337653033383637376532
+38613633353535316631343931626661616535313133303433383663376138396464633961643435
+63666137396330653461323737636532303064646434326136613666386430343966313733323433
+36636632633635353134626630326235373466393661306333353431666335646333386665613333
+38653831663636393261326639643931363633633835353436373832316665386235366463373438
+61643465633730316334396231633337333533663461343432396631393035383035376338663438
+65323765366335613062353863616237373066346332633864636535393638386236393163666266
+34393163386232633034366637306333363130646136363630376134396539373439306461306333
+63356338646132623031356638313434393638623935633038386662376237666362326134343137
+66393661626562656263303235646530303839383562643437313334356464666532343662303237
+66333139333961663733663462626430633365303031633736393731646239326236353839343162
+66656263613866383666663839366234333830666261303665333662333434333432633662363232
+38363430343863343733626662613931386266333261303263613865636437353233343530336331
+37316535633765346430303439333235653139386438656562613331363663336637656237666539
+64366362646236303738663062646539306430353838653639323331313237303533303932373964
+61616637303562396237636238626139376139643430366162613665363830343736646333396634
+30323435336566343265336166303361623435323037366136633864616532326531306164393532
+38343639393539373839393261613966643239303961323137353361656161326564373035383461
+38653934363262613661346639326232306564303434356635656232343632376237633334613863
+62386630373261306339373065366537333033616264363265303539333132336531393135326262
+38366232326536333435656337363638396133366234336164353737336362643034623161646461
+34623262613333646561666135303061653164633266383030646364313437623834313735373863
+65623834646533343338343132373962623739393830623230663130316239623564383563663533
+33303964353462663734656538356336326437313534383062366564373631353764646262313934
+62663637303565623562373432613664646439346631666235373139366335343065313633643164
+31386639373635616136303636363034363362386134656639363166363136313939316530363666
+61666635656231393965663630373961623631653831393631623634376138643930663466613732
+34333465633762303065333162653366363333313962626563373236303432623164656565383533
+31366530353539383665336265326238393133366538396266656463393739373036313966646136
+36333261613865316431653233626561366531663335306339316538346337386639376161623038
+30303665383064323162393432633530303162333865356337373131623834353230363265313734
+62613263323630666336383839323639383262323938363531323131646363343136353664313663
+63396263353139393037343834333665303135656366663637653662616235376437303238306434
+62323633373063643861373833343762373039666233343866396531326639376632366239613133
+34356266343035323265363534643831373166336364653232353766646262613337653435323330
+30653538316264313933623563386462643832356334393032613465383737616337636131356332
+36313130323066366537356136393431396263306136663038366536336165376462616562616362
+66313132313665663366356230313965373335326332366433633031386139373061643335386231
+63396431393630623432313130316364353835663935663064303639613263366236653261333861
+33616162613335363163393661623766366565343539303734333461306564376139303336303835
+62333362643365326338623262653961353933386563643438366463326132376537346337383461
+39323230366533646630376338353263623335306330326334326233326666616537653164656632
+36626231366132306533323132323134333337303731323133313933376231376662653133653964
+37346139663162313731653261353630313866616630643738373766666564373033643333306665
+36626235623762313362313035623939373435333366323135363034366262336239616537626338
+31653763666663383637663538626236303761343934366635636538663166663261666263303230
+33323565376434323632336532633664316664616564643465653831633562303032393131333531
+36373964313863656663343461313931336431333864343332333038383533353339666532333730
+30356364353662383034313131326431663463656162383533356235666163636262326437383737
+33363739643533333663326132333333356331336230383738646464366333363561616435643830
+62333037366462316564306366366461616136363963616233383531636465373731326438316565
+37313630613836306465306238336435616261343263313739373664343363656264633764393030
+32316431616431333537383037386231363139643564363462316338303461336631363463636330
+32663632346239313634633261613165613433313731613164623165636234393434353132386562
+35353239653638333632656336643338643363613833366163333338343364656461353631666630
+37643535643662653635613761376133366363633936366363353130613132366532643164613864
+64383264306235666663346166613561363461376233336133663962376563663761303434613063
+65623630383430656536326238363632313539306332643231633131326133343731326337623036
+36346665636532316363393339636438613133363937653436656333336465353035393064316665
+62613635376664366363646663633330343034313037633665653565353931633235346438323631
+64626134326666613330643432346633396464613730346438623937613565613030626361663730
+64366334623563656566393865613964613136623562396366383964646538356166393461623530
+34633430626637633636313866316463636431393030336630623733393031616564363161653830
+37383136363631643661356239613431383331383639393064353734653561326262636466343838
+34616330343631356536386634383164616531306264343630666163326261303166613366363235
+64306464306631383633306264626338666439643562373966396332663937663839303965333733
+34343033363665353763613463663363353735613635393263636135376430363062393635363864
+65313666363864666263623634326661636531366435616637303535363731306131343761313663
+30363639386564323965333738383236626334653464373331333062666230353834323062343236
+30663431313537623661633263366435393632383030633631616631363732646431323135643738
+30363561343637393761376130323034323831666535303563313130623664666439383539633234
+39636665633361646537653766333963333636313764383363666665626663353264613565376439
+38373362373239303131343131326333666230393433373734393431353537656334303031663365
+346132616235663836643932323633663662

group_vars/stage_prodwork01/backup.yml

@@ -1,41 +1,201 @@
 backup_lvm_hcloudvol_size: 30
 backup_lvm_hcloudvol_count: 8
 
-minio_nsodev_accesskey: "{{ minio_nsodev_accesskey_vault }}"
+backup_user_ssh_privkey: "{{ backup_user_ssh_privkey_vault }}"
-minio_nsodev_secretkey: "{{ minio_nsodev_secretkey_vault }}"
 
-minio_cusqa_accesskey: "{{ minio_cusqa_accesskey_vault }}"
+# Admin access for S3 Storage on stage nsodev
-minio_cusqa_secretkey: "{{ minio_cusqa_secretkey_vault }}"
+nsodev_minio_admin_accesskey: "{{ nsodev_minio_admin_accesskey_vault }}"
+nsodev_minio_admin_secretkey: "{{ nsodev_minio_admin_secretkey_vault }}"
 
-minio_cusprod_accesskey: "{{ minio_cusprod_accesskey_vault }}"
+# Readonly access for S3 Storage on stage nsodev all buckets
-minio_cusprod_secretkey: "{{ minio_cusprod_secretkey_vault }}"
+nsodev_minio_user_read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey_vault }}"
+nsodev_minio_user_read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage nsodev bucket wordpress 
+nsodev_minio_user_read_write_wordpress_accesskey: "{{ nsodev_minio_user_read_write_wordpress_accesskey_vault }}"
+nsodev_minio_user_read_write_wordpress_secretkey: "{{ nsodev_minio_user_read_write_wordpress_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage nsodev bucket postgres 
+nsodev_minio_user_read_write_postgres_accesskey: "{{ nsodev_minio_user_read_write_postgres_accesskey_vault }}"
+nsodev_minio_user_read_write_postgres_secretkey: "{{ nsodev_minio_user_read_write_postgres_secretkey_vault }}"
+
+
+# Admin access for S3 Storage on stage cusqa
+cusqa_minio_admin_accesskey: "{{ cusqa_minio_admin_accesskey_vault }}"
+cusqa_minio_admin_secretkey: "{{ cusqa_minio_admin_secretkey_vault }}"
+
+# Readonly access for S3 Storage on stage cusqa all buckets
+cusqa_minio_user_read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey_vault }}"
+cusqa_minio_user_read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage cusqa bucket wordpress 
+cusqa_minio_user_read_write_wordpress_accesskey: "{{ cusqa_minio_user_read_write_wordpress_accesskey_vault }}"
+cusqa_minio_user_read_write_wordpress_secretkey: "{{ cusqa_minio_user_read_write_wordpress_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage cusqa bucket postgres 
+cusqa_minio_user_read_write_postgres_accesskey: "{{ cusqa_minio_user_read_write_postgres_accesskey_vault }}"
+cusqa_minio_user_read_write_postgres_secretkey: "{{ cusqa_minio_user_read_write_postgres_secretkey_vault }}"
+
+
+# Admin access for S3 Storage on stage cusprod
+cusprod_minio_admin_accesskey: "{{ cusprod_minio_admin_accesskey_vault }}"
+cusprod_minio_admin_secretkey: "{{ cusprod_minio_admin_secretkey_vault }}"
+
+# Readonly access for S3 Storage on stage cusprod all buckets
+cusprod_minio_user_read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey_vault }}"
+cusprod_minio_user_read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage cusprod bucket wordpress 
+cusprod_minio_user_read_write_wordpress_accesskey: "{{ cusprod_minio_user_read_write_wordpress_accesskey_vault }}"
+cusprod_minio_user_read_write_wordpress_secretkey: "{{ cusprod_minio_user_read_write_wordpress_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage cusprod bucket postgres 
+cusprod_minio_user_read_write_postgres_accesskey: "{{ cusprod_minio_user_read_write_postgres_accesskey_vault }}"
+cusprod_minio_user_read_write_postgres_secretkey: "{{ cusprod_minio_user_read_write_postgres_secretkey_vault }}"
+
+
+# Admin access for S3 Storage on stage keycloak
+keycloak_minio_admin_accesskey: "{{ keycloak_minio_admin_accesskey_vault }}"
+keycloak_minio_admin_secretkey: "{{ keycloak_minio_admin_secretkey_vault }}"
+
+# Readonly access for S3 Storage on stage keycloak all buckets
+keycloak_minio_user_read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey_vault }}"
+keycloak_minio_user_read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey_vault }}"
+
+# ReadWrite access for S3 Storage on stage keycloak bucket postgres 
+keycloak_minio_user_read_write_postgres_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey_vault }}"
+keycloak_minio_user_read_write_postgres_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey_vault }}"
 
-minio_keycloak_accesskey: "{{ minio_keycloak_accesskey_vault }}"
-minio_keycloak_secretkey: "{{ minio_keycloak_secretkey_vault }}"
-minio_keycloak_url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital"
 
 minio_stage_dicts:
   - {
       stage: "nsodev",
       url: "https://s3storage-nsodev-prodwork01.smardigo.digital",
-      minio_accesskey: "{{ minio_nsodev_accesskey }}",
+      read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey }}",
-      minio_secretkey: "{{ minio_nsodev_secretkey }}",
+      read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ nsodev_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ nsodev_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ nsodev_minio_admin_accesskey }}",
+      admin_secretkey: "{{ nsodev_minio_admin_secretkey }}",
       hour: "3",
       minute: "30",
+      bucket: "postgres"
+    }
+  - {
+      stage: "nsodev",
+      url: "https://s3storage-nsodev-prodwork01.smardigo.digital",
+      read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ nsodev_minio_user_read_write_wordpress_accesskey }}",
+      read_write_secretkey: "{{ nsodev_minio_user_read_write_wordpress_secretkey }}",
+      admin_accesskey: "{{ nsodev_minio_admin_accesskey }}",
+      admin_secretkey: "{{ nsodev_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "35",
+      bucket: "wordpress"
     }
   - {
       stage: "cusqa",
       url: "https://s3storage-cusqa-prodwork01.smardigo.digital",
-      minio_accesskey: "{{ minio_cusqa_accesskey }}",
+      read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey }}",
-      minio_secretkey: "{{ minio_cusqa_secretkey }}",
+      read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ cusqa_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ cusqa_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ cusqa_minio_admin_accesskey }}",
+      admin_secretkey: "{{ cusqa_minio_admin_secretkey }}",
       hour: "3",
-      minute: "30",
+      minute: "40",
+      bucket: "postgres"
+    }
+  - {
+      stage: "cusqa",
+      url: "https://s3storage-cusqa-prodwork01.smardigo.digital",
+      read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ cusqa_minio_user_read_write_wordpress_accesskey }}",
+      read_write_secretkey: "{{ cusqa_minio_user_read_write_wordpress_secretkey }}",
+      admin_accesskey: "{{ cusqa_minio_admin_accesskey }}",
+      admin_secretkey: "{{ cusqa_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "45",
+      bucket: "wordpress"
+    }
+  - {
+      stage: "cusprod",
+      url: "https://s3storage-cusprod-prodwork01.smardigo.digital",
+      read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ cusprod_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ cusprod_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ cusprod_minio_admin_accesskey }}",
+      admin_secretkey: "{{ cusprod_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "50",
+      bucket: "postgres"
     }
   - {
       stage: "cusprod",
       url: "https://s3storage-cusprod-prodwork01.smardigo.digital",
-      minio_accesskey: "{{ minio_cusprod_accesskey }}",
+      read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey }}",
-      minio_secretkey: "{{ minio_cusprod_secretkey }}",
+      read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ cusprod_minio_user_read_write_wordpress_accesskey }}",
+      read_write_secretkey: "{{ cusprod_minio_user_read_write_wordpress_secretkey }}",
+      admin_accesskey: "{{ cusprod_minio_admin_accesskey }}",
+      admin_secretkey: "{{ cusprod_minio_admin_secretkey }}",
+      hour: "3",
+      minute: "55",
+      bucket: "wordpress"
+    }
+  - {
+      stage: "keycloak",
+      url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital",
+      read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey }}",
+      read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey }}",
+      read_write_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey }}",
+      read_write_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey }}",
+      admin_accesskey: "{{ keycloak_minio_admin_accesskey }}",
+      admin_secretkey: "{{ keycloak_minio_admin_secretkey }}",
       hour: "4",
       minute: "0",
+      bucket: "postgres"
     }
+
+
+# minio_nsodev_accesskey: "{{ minio_nsodev_accesskey_vault }}"
+# minio_nsodev_secretkey: "{{ minio_nsodev_secretkey_vault }}"
+
+# minio_cusqa_accesskey: "{{ minio_cusqa_accesskey_vault }}"
+# minio_cusqa_secretkey: "{{ minio_cusqa_secretkey_vault }}"
+
+# minio_cusprod_accesskey: "{{ minio_cusprod_accesskey_vault }}"
+# minio_cusprod_secretkey: "{{ minio_cusprod_secretkey_vault }}"
+
+# minio_keycloak_accesskey: "{{ minio_keycloak_accesskey_vault }}"
+# minio_keycloak_secretkey: "{{ minio_keycloak_secretkey_vault }}"
+# minio_keycloak_url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital"
+
+# minio_stage_dicts:
+#   - {
+#       stage: "nsodev",
+#       url: "https://s3storage-nsodev-prodwork01.smardigo.digital",
+#       minio_accesskey: "{{ minio_nsodev_accesskey }}",
+#       minio_secretkey: "{{ minio_nsodev_secretkey }}",
+#       hour: "3",
+#       minute: "30",
+#     }
+#   - {
+#       stage: "cusqa",
+#       url: "https://s3storage-cusqa-prodwork01.smardigo.digital",
+#       minio_accesskey: "{{ minio_cusqa_accesskey }}",
+#       minio_secretkey: "{{ minio_cusqa_secretkey }}",
+#       hour: "3",
+#       minute: "30",
+#     }
+#   - {
+#       stage: "cusprod",
+#       url: "https://s3storage-cusprod-prodwork01.smardigo.digital",
+#       minio_accesskey: "{{ minio_cusprod_accesskey }}",
+#       minio_secretkey: "{{ minio_cusprod_secretkey }}",
+#       hour: "4",
+#       minute: "0",
+#     }

pmci-server-create.yml

@@ -94,6 +94,7 @@
           'docker-logrotate',
           'docker-engine',
           'smartmontools',
+          'mc',
         ]
         state: 'absent'
       when: ansible_distribution == "Ubuntu"

roles/backup/files/mirror_bucket_from_minio_server.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Script:  mirror_bucket_from_minio_server.sh <MINIO_URL> <STAGE> <ACCESSKEY> <SECRETKEY> <BUCKET>
+#          This script is run as a cron job.
+#          It mirrors an S3 bucket to the local backup directory.
+#          A readonly user is used for S3 access.
+# Example: mirror_bucket_from_minio_server.sh https://s3storage-minio-devnso.smardigo.digital devnso minio-readonly-devnso secretkey wordpress
+# Version: 1.0
+# Author:  ext.Hans-Peter.Wissenbach@netgo.de
+# History: 2023.07.27 - rework previous script pull_from_minio_server.sh and add bucket
+#          2023.08.09 - add mirror result and error detection
+
+MINIO_URL=$1
+STAGE=$2
+ACCESSKEY=$3
+SECRETKEY=$4
+BUCKET=$5
+
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/bucket/${BUCKET}"
+METRICS_FILE="${HOME}/metrics_${STAGE}_${BUCKET}.prom"
+ALIAS="${STAGE}"
+
+BACKUP_START=$(date +%s)
+
+mkdir -p ${LOCAL_BACKUP_DIR}
+
+rm -rf ${LOCAL_BACKUP_DIR}/*
+
+mcli alias set ${ALIAS} ${MINIO_URL} ${ACCESSKEY} ${SECRETKEY}
+RC="$?"
+
+if [[ ${RC} -eq "0" ]]
+then
+  RESULT=$(mcli mirror ${ALIAS}/${BUCKET} ${LOCAL_BACKUP_DIR} --overwrite --newer-than 1d --json)
+  echo "${RESULT}"
+  TRANSFERRED=$(echo "${RESULT}" | jq '.transferred | select(. != null)')
+  TRANSFERRED="${TRANSFERRED:=0}" # default 0 if not set
+  STATUS=$(echo "${RESULT}" | jq -e '.status == "success"')
+  RC="$?"
+
+  mcli alias rm ${ALIAS}
+fi
+
+BACKUP_END=$(date +%s)
+
+if [[ ${RC} -eq "0" ]] && [[ ${TRANSFERRED} > 0 ]]
+then
+  echo "Nightly Backup Successful - writing METRICS_FILE: ${METRICS_FILE}"
+  tee <<EOF > ${METRICS_FILE}
+# HELP nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970).
+# TYPE nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} gauge
+nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_START}
+
+# HELP nightly_backup_successful_${STAGE}_${BUCKET}
+# TYPE nightly_backup_successful_${STAGE}_${BUCKET} gauge
+nightly_backup_successful_${STAGE}_${BUCKET}{stage="${STAGE}"} 0
+
+# HELP nightly_backup_transferred_bytes_${STAGE}_${BUCKET}
+# TYPE nightly_backup_transferred_bytes_${STAGE}_${BUCKET} gauge
+nightly_backup_transferred_bytes_${STAGE}_${BUCKET}{stage="${STAGE}"} ${TRANSFERRED}
+
+# HELP nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970).
+# TYPE nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} gauge
+nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_END}
+EOF
+else
+  echo "Nightly Backup Failed - writing METRICS_FILE: ${METRICS_FILE}"
+  tee <<EOF > ${METRICS_FILE}
+# HELP nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970).
+# TYPE nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} gauge
+nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_START}
+
+# HELP nightly_backup_successful_${STAGE}_${BUCKET}
+# TYPE nightly_backup_successful_${STAGE}_${BUCKET} gauge
+nightly_backup_successful_${STAGE}_${BUCKET}{stage="${STAGE}"} 1
+
+# HELP nightly_backup_transferred_bytes_${STAGE}_${BUCKET}
+# TYPE nightly_backup_transferred_bytes_${STAGE}_${BUCKET} gauge
+nightly_backup_transferred_bytes_${STAGE}_${BUCKET}{stage="${STAGE}"} ${TRANSFERRED}
+
+# HELP nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970).
+# TYPE nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} gauge
+nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_END}
+EOF
+fi
+
+exit ${RC}

roles/backup/files/read_only_policy.json

@@ -0,0 +1,16 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+            "s3:ListBucket",
+            "s3:GetObject"
+        ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:s3:::*"
+      ],
+      "Sid": "ReadOnlyPolicy"
+    }
+  ]
+}

roles/backup/files/read_write_postgres_policy.json

@@ -0,0 +1,18 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+            "s3:ListBucket",
+            "s3:GetObject",
+            "s3:PutObject",
+            "s3:DeleteObject"
+        ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:s3:::postgres/*", "arn:aws:s3:::postgres"
+      ],
+      "Sid": "ReadWritePostgresPolicy"
+    }
+  ]
+}

roles/backup/files/read_write_wordpress_policy.json

@@ -0,0 +1,18 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+            "s3:ListBucket",
+            "s3:GetObject",
+            "s3:PutObject",
+            "s3:DeleteObject"
+        ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:s3:::wordpress/*", "arn:aws:s3:::wordpress"
+      ],
+      "Sid": "ReadWriteWordpressPolicy"
+    }
+  ]
+}

roles/backup/tasks/main.yml

@@ -41,7 +41,7 @@
     group: '{{ system_user }}'
     content: '{{ backup_user_ssh_privkey_vault }}'
 
-- name: "Providing rsync script"
+- name: "Providing Backup scripts"
   become: yes
   copy:
     src: '{{ item }}'
@@ -52,6 +52,10 @@
   with_items:
     - pull_remote_backups.sh
     - push_backups_to_restore_server.sh
+    - mirror_bucket_from_minio_server.sh
+    - read_only_policy.json
+    - read_write_postgres_policy.json
+    - read_write_wordpress_policy.json
 
 - name: Touch metrics.prom if not exists
   file:
@@ -94,3 +98,124 @@
     src: "/home/{{ system_user }}/backup_status_postgres.prom"
     dest: "/var/lib/prometheus/node-exporter/backup_status_postgres.prom"
     state: link
+  
+- name: Recursively change ownership of backups directory
+  ansible.builtin.file:
+    path: /home/{{ system_user }}/backups
+    state: directory
+    recurse: yes
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Download minio client
+  become: yes
+  ansible.builtin.get_url:
+    url: https://dl.min.io/client/mc/release/linux-amd64/mc
+    dest: /usr/bin/mcli
+    mode: '0755'
+
+- name: "Set MinIO alias for {{ item.stage }}_admin" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli alias set {{ item.stage }}_admin {{ item.url }} {{ item.admin_accesskey }} {{ item.admin_secretkey }}'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: "Add MinIO read only users {{ item.read_only_accesskey }}" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin user add {{ item.stage }}_admin {{ item.read_only_accesskey }} {{ item.read_only_secretkey }}'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: "Add MinIO read write user {{ item.read_write_accesskey }}" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin user add {{ item.stage }}_admin {{ item.read_write_accesskey }} {{ item.read_write_secretkey }}'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: "Create MinIO read only policy" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin policy create {{ item.stage }}_admin read_only_policy /home/{{ system_user }}/read_only_policy.json'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: "Attach MinIO read only policy to user {{ item.read_only_accesskey }}" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin policy attach {{ item.stage }}_admin read_only_policy --user {{ item.read_only_accesskey }}'
+  loop: "{{ minio_stage_dicts }}"
+  register: policy_read_only_result
+  failed_when: "'policy is already attached' not in policy_read_only_result.stderr and policy_read_only_result.rc == 1"
+  
+- name: "Create MinIO read write policy per bucket" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin policy create {{ item.stage }}_admin read_write_{{ item.bucket }}_policy /home/{{ system_user }}/read_write_{{ item.bucket }}_policy.json'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: "Attach MinIO read write policy to user {{ item.read_write_accesskey }}" # noqa command-instead-of-shell no-changed-when
+  become: true
+  become_user: '{{ system_user }}'
+  ansible.builtin.shell: 'mcli admin policy attach {{ item.stage }}_admin read_write_{{ item.bucket }}_policy --user {{ item.read_write_accesskey }}'
+  loop: "{{ minio_stage_dicts }}"
+  register: policy_read_write_result
+  failed_when: "'policy is already attached' not in policy_read_write_result.stderr and policy_read_write_result.rc == 1"
+
+# wird abgelöst durch mirror_bucket_from_minio_server.sh
+# - name: Create Cron Job for pull_from_minio_server.sh script
+#   ansible.builtin.cron:
+#     name: "pull minio backups for {{ item.stage }}"
+#     hour: "{{ item.hour }}"
+#     minute: "{{ item.minute }}"
+#     user: '{{ system_user }}'
+#     job: "/home/{{ system_user }}/pull_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.minio_accesskey }} {{ item.minio_secretkey }}"
+#   loop: "{{ minio_stage_dicts }}"
+
+# wird abgelöst durch mirror_bucket_from_minio_server.sh
+# - name: Create Cron Job for keycloak_pull_from_minio_server.sh script
+#   ansible.builtin.cron:
+#     name: "pull minio backups for keycloak"
+#     hour: "2"
+#     minute: "30"
+#     user: '{{ system_user }}'
+#     job: "/home/{{ system_user }}/keycloak_pull_from_minio_server.sh {{ minio_keycloak_url }} {{ minio_keycloak_accesskey }} {{ minio_keycloak_secretkey }}"
+
+- name: "Create Cron Job for each bucket with mirror_bucket_from_minio_server.sh script"
+  ansible.builtin.cron:
+    name: "pull minio backups for {{ item.stage }} and bucket {{ item.bucket }}"
+    hour: "{{ item.hour }}"
+    minute: "{{ item.minute }}"
+    user: '{{ system_user }}'
+    job: "/home/{{ system_user }}/mirror_bucket_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.read_only_accesskey }} {{ item.read_only_secretkey }} {{ item.bucket }}"
+  loop: "{{ minio_stage_dicts }}"
+
+- name: Touch metrics_{{ item.stage }}_{{ item.bucket }}.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/metrics_{{ item.stage }}_{{ item.bucket }}.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+  loop: "{{ minio_stage_dicts }}"
+
+- name: Create symbolic link for node_exporter text {{ item.stage }} metrics
+  file:
+    src: "/home/{{ system_user }}/metrics_{{ item.stage }}_{{ item.bucket }}.prom"
+    dest: "/var/lib/prometheus/node-exporter/metrics_{{ item.stage }}_{{ item.bucket }}.prom"
+    state: link
+  loop: "{{ minio_stage_dicts }}"
+
+# wird nicht mehr benötigt wenn umgestellt auf mirror_bucket_from_minio_server.sh
+# - name: Touch metrics_keycloak.prom if not exists
+#   file:
+#     path: "/home/{{ system_user }}/metrics_keycloak.prom"
+#     state: touch
+#     mode: '0744'
+#     owner: '{{ system_user }}'
+#     group: '{{ system_user }}'
+
+# wird nicht mehr benötigt wenn umgestellt auf mirror_bucket_from_minio_server.sh
+# - name: Create symbolic link for node_exporter text nsodev metrics
+#   file:
+#     src: "/home/{{ system_user }}/metrics_keycloak.prom"
+#     dest: "/var/lib/prometheus/node-exporter/metrics_keycloak.prom"
+#     state: link

roles/backup_minio/tasks/main.yml

@@ -1,119 +0,0 @@
----
-
-- name: "Backup storage server | create system user"
-  become: yes
-  ansible.builtin.user:
-    name: '{{ system_user }}'
-    comment: "user for backup"
-    shell: /bin/bash
-  register: create_user
-
-- name: "Create .ssh dir and backups dir"
-  become: yes
-  file:
-    path: '/home/{{ system_user }}/{{ item.name }}/'
-    mode: '{{ item.mode }}'
-    owner: '{{ system_user }}'
-    group: '{{ system_user }}'
-    state: directory
-  loop:
-    - name: '.ssh'
-      mode: '0700'
-    - name: 'backups'
-      mode: '0775'
-
-- name: "Create/Resize LVM for datadir"
-  include_role:
-    name: lvm_with_hetzner_volumes
-  vars:
-    lvm_with_hetzner_volumes__volprefix: backup_datadir
-    lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}"
-    lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}"
-    lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}"
-
-- name: Recursively change ownership of backups directory
-  ansible.builtin.file:
-    path: /home/{{ system_user }}/backups
-    state: directory
-    recurse: yes
-    owner: '{{ system_user }}'
-    group: '{{ system_user }}'
-
-- name: Download minio client
-  become: yes
-  ansible.builtin.get_url:
-    url: https://dl.min.io/client/mc/release/linux-amd64/mc
-    dest: /usr/bin/mc
-    mode: '0755'
-
-# - name: "Providing SSH priv.key"
-#   no_log: true
-#   become: yes
-#   copy:
-#     dest: '/home/{{ system_user }}/.ssh/id_rsa'
-#     mode: '0400'
-#     owner: '{{ system_user }}'
-#     group: '{{ system_user }}'
-#     content: '{{ backup_user_ssh_privkey_vault }}'
-
-- name: "Providing mc client script"
-  become: yes
-  copy:
-    src: '{{ item }}'
-    dest: '/home/{{ system_user }}/{{ item }}'
-    mode: '0755'
-    owner: '{{ system_user }}'
-    group: '{{ system_user }}'
-  with_items:
-    - pull_from_minio_server.sh
-    - keycloak_pull_from_minio_server.sh
-
-- name: Create Cron Job for pull_from_minio_server.sh script
-  ansible.builtin.cron:
-    name: "pull minio backups for {{ item.stage }}"
-    hour: "{{ item.hour }}"
-    minute: "{{ item.minute }}"
-    user: '{{ system_user }}'
-    job: "/home/{{ system_user }}/pull_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.minio_accesskey }} {{ item.minio_secretkey }}"
-  loop: "{{ minio_stage_dicts }}"
-
-- name: Create Cron Job for keycloak_pull_from_minio_server.sh script
-  ansible.builtin.cron:
-    name: "pull minio backups for keycloak"
-    hour: "2"
-    minute: "30"
-    user: '{{ system_user }}'
-    job: "/home/{{ system_user }}/keycloak_pull_from_minio_server.sh {{ minio_keycloak_url }} {{ minio_keycloak_accesskey }} {{ minio_keycloak_secretkey }}"
-
-- name: Touch metrics_nsodev.prom if not exists
-  file:
-    path: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom"
-    state: touch
-    mode: '0744'
-    owner: '{{ system_user }}'
-    group: '{{ system_user }}'
-  loop: "{{ minio_stage_dicts }}"
-
-
-- name: Create symbolic link for node_exporter text nsodev metrics
-  file:
-    src: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom"
-    dest: "/var/lib/prometheus/node-exporter/metrics_{{ item.stage }}.prom"
-    state: link
-  loop: "{{ minio_stage_dicts }}"
-
-- name: Touch metrics_keycloak.prom if not exists
-  file:
-    path: "/home/{{ system_user }}/metrics_keycloak.prom"
-    state: touch
-    mode: '0744'
-    owner: '{{ system_user }}'
-    group: '{{ system_user }}'
-
-
-- name: Create symbolic link for node_exporter text nsodev metrics
-  file:
-    src: "/home/{{ system_user }}/metrics_keycloak.prom"
-    dest: "/var/lib/prometheus/node-exporter/metrics_keycloak.prom"
-    state: link
-

roles/backup_old/files/pull_remote_backups.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+#
+#
+
+# Fail fast and be aware of exit codes
+set -euo pipefail
+
+# Define some variables
+DATE=$(date +%F)
+DATE_TIME=$(date +%F_%H:%M)
+REMOTE_SYSTEM_USER=backupuser
+DATABASE_SERVER=$1
+STAGE=$2
+DATABASE_ENGINE=$3
+DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}
+METRICS_FILE=${HOME}/backup_status_${DATABASE_ENGINE}.prom
+LOG_FILE=${DEST_DIR}/backup_${DATE_TIME}.log
+
+# Create backup directory ${DEST_DIR} if not exist
+mkdir -p ${DEST_DIR}
+
+# Redirect stderr to stdout and save everything to log file
+exec > ${LOG_FILE} 2>&1
+
+# Log backup sync start time
+echo "----- Start backup Sync - ${DATE_TIME} -----"
+
+# Remove files oder than 48h in ${DEST_DIR}
+find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR"
+[ "$?" != "0" ] && exit 1
+
+echo "Removing logfiles older than 7d ..."
+find $DEST_DIR -type f -mtime +7 -name "backup_*.log" -print0 | xargs -I OLD_FILES -0 rm -rf "OLD_FILES"
+
+# Start rsync job from ${DATABASE_SERVER} to ${DEST_DIR}/
+rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/
+[ "$?" -eq "0" ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1"
+
+BACKUP_STATUS_FILE=$(ls -t1 ${DEST_DIR}/${DATE}/backup_finished_${DATE}_* | head -n1)
+# Check existence of current ${BACKUP_STATUS_FILE}, which is created by AWX, in case of succesful database backup only.
+[ -f ${BACKUP_STATUS_FILE} ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1"
+
+# Add backup status to Prometheus metrics file
+if [ "$NIGHTLY_BACKUP_SUCCESSFUL" -eq "0" ]; then
+  echo "NIGHTLY_BACKUP_SUCCESSFUL=0 - writing METRICS_FILE"
+cat <<EOF > $METRICS_FILE
+# HELP nightly_backup_successful_${DATABASE_ENGINE}
+# TYPE nightly_backup_successful_${DATABASE_ENGINE} gauge
+nightly_backup_successful_${DATABASE_ENGINE}{stage="$STAGE"} $NIGHTLY_BACKUP_SUCCESSFUL
+nightly_backup_successful_${DATABASE_ENGINE}_finished_seconds{stage="$STAGE"} `date +%s`
+EOF
+
+else
+  echo "NIGHTLY_BACKUP_SUCCESSFUL=1 - removing METRICS_FILE to trigger alert"
+  rm $METRICS_FILE
+fi
+
+# Log backup sync end time
+echo "----- End backup Sync - ${DATE_TIME} -----"

roles/backup_old/files/push_backups_to_restore_server.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+#
+#
+
+REMOTE_SYSTEM_USER=backupuser
+RESTORE_SERVER=$1
+DATABASE_SERVER=$2
+STAGE=$3
+DATABASE_ENGINE=$4
+
+# currently it defaults to todays date
+DATE=$(date +%F)
+
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}"
+BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1)
+
+REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}"
+DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"
+
+if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then
+   echo "BACKUP_FILE_FOR_TRANSFER not found. EXIT" && exit 1
+fi
+
+# avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process
+ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${RESTORE_SERVER}
+
+SSH_OPTIONS='-o StrictHostKeyChecking=no'
+
+# needed due to unknown rsync option --mkpath in rsync version 3.1.3
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "mkdir -p ${DEST_DIR}"
+
+rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER}:${DEST_DIR}
+
+BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}')
+
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}"
+

roles/backup_old/tasks/main.yml

@@ -0,0 +1,96 @@
+---
+
+- name: "Backup storage server | create system user"
+  become: yes
+  ansible.builtin.user:
+    name: '{{ system_user }}'
+    comment: "user for backup"
+    shell: /bin/bash
+  register: create_user
+
+- name: "Create .ssh dir and backups dir"
+  become: yes
+  file:
+    path: '/home/{{ system_user }}/{{ item.name }}/'
+    mode: '{{ item.mode }}'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    state: directory
+  loop:
+    - name: '.ssh'
+      mode: '0700'
+    - name: 'backups'
+      mode: '0775'
+
+- name: "Create/Resize LVM for datadir"
+  include_role:
+    name: lvm_with_hetzner_volumes
+  vars:
+    lvm_with_hetzner_volumes__volprefix: backup_datadir
+    lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}"
+    lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}"
+    lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}"
+
+- name: "Providing SSH priv.key"
+  no_log: true
+  become: yes
+  copy:
+    dest: '/home/{{ system_user }}/.ssh/id_rsa'
+    mode: '0400'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    content: '{{ backup_user_ssh_privkey_vault }}'
+
+- name: "Providing Backup scripts"
+  become: yes
+  copy:
+    src: '{{ item }}'
+    dest: '/home/{{ system_user }}/{{ item }}'
+    mode: '0755'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+  with_items:
+    - pull_remote_backups.sh
+    - push_backups_to_restore_server.sh
+
+- name: Touch metrics.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/metrics.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Touch backup_status_maria.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/backup_status_maria.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Touch backup_status_postgres.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/backup_status_postgres.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Create symbolic link for node_exporter text metrics
+  file:
+    src: "/home/{{ system_user }}/metrics.prom"
+    dest: "/var/lib/prometheus/node-exporter/offsite-metrics.prom"
+    state: link
+
+- name: Create symbolic link for node_exporter text metrics backup_status_maria
+  file:
+    src: "/home/{{ system_user }}/backup_status_maria.prom"
+    dest: "/var/lib/prometheus/node-exporter/backup_status_maria.prom"
+    state: link
+
+- name: Create symbolic link for node_exporter text metrics backup_status_postgres
+  file:
+    src: "/home/{{ system_user }}/backup_status_postgres.prom"
+    dest: "/var/lib/prometheus/node-exporter/backup_status_postgres.prom"
+    state: link

setup.yml

@@ -37,6 +37,7 @@
           'docker-logrotate',
           'docker-engine',
           'smartmontools',
+          'mc',
         ]
         state: 'absent'
       when: ansible_distribution == "Ubuntu"

smardigo.yml

@@ -67,9 +67,6 @@
     - role: backup
       when: "'backup' in group_names"
 
-    - role: backup_minio
-      when: "'backup_minio' in group_names"
-
     - role: keycloak_compact
       when: "'keycloak_compact' in group_names"
 

stage-prodwork01

@@ -1,7 +1,4 @@
-[postfix]
+[backup]
-prodwork01-mail-01
-
-[backup_minio]
 prodwork01-backup-01
 
 [kube_control_plane]
@@ -30,7 +27,7 @@ kube_node
 [stage_prodwork01:children]
 postfix
 k8s_cluster
-backup_minio
+backup
 
 [all:children]
 stage_prodwork01