diff --git a/create-server.yml b/create-server.yml index d1a9ae1..67b8da1 100644 --- a/create-server.yml +++ b/create-server.yml @@ -119,6 +119,7 @@ 'docker-logrotate', 'docker-engine', 'smartmontools', + 'mc', ] state: 'absent' when: ansible_distribution == "Ubuntu" diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index da34c21..88c6c26 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -34,7 +34,7 @@ metricbeat_enabled: false node_exporter_enabled: true common_apt_dependencies: - - mc + - jq - vim # TODO Check if we really want this - zip diff --git a/group_vars/stage_devnso/backup.yml b/group_vars/stage_devnso/backup.yml new file mode 100644 index 0000000..2c6efb3 --- /dev/null +++ b/group_vars/stage_devnso/backup.yml @@ -0,0 +1,74 @@ +# Backup space +#backup_lvm_hcloudvol_size: 30 +#backup_lvm_hcloudvol_count: 8 + +backup_user_ssh_privkey: "{{ backup_user_ssh_privkey_vault }}" + +# Admin access for S3 Storage on stage devnso +devnso_minio_admin_accesskey: "{{ devnso_minio_admin_accesskey_vault }}" +devnso_minio_admin_secretkey: "{{ devnso_minio_admin_secretkey_vault }}" + +# Readonly access for S3 Storage on stage devnso all buckets +devnso_minio_user_read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey_vault }}" +devnso_minio_user_read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage devnso bucket wordpress +devnso_minio_user_read_write_wordpress_accesskey: "{{ devnso_minio_user_read_write_wordpress_accesskey_vault }}" +devnso_minio_user_read_write_wordpress_secretkey: "{{ devnso_minio_user_read_write_wordpress_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage devnso bucket postgres +devnso_minio_user_read_write_postgres_accesskey: "{{ devnso_minio_user_read_write_postgres_accesskey_vault }}" +devnso_minio_user_read_write_postgres_secretkey: "{{ devnso_minio_user_read_write_postgres_secretkey_vault }}" + +# Admin access for S3 Storage on "stage" keycloak +keycloak_minio_admin_accesskey: "{{ keycloak_minio_admin_accesskey_vault }}" +keycloak_minio_admin_secretkey: "{{ keycloak_minio_admin_secretkey_vault }}" + +# Readonly access for S3 Storage on "stage" keycloak all buckets +keycloak_minio_user_read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey_vault }}" +keycloak_minio_user_read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on "stage" keycloak bucket postgres +keycloak_minio_user_read_write_postgres_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey_vault }}" +keycloak_minio_user_read_write_postgres_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey_vault }}" + +minio_stage_dicts: + - { + stage: "devnso", + url: "https://s3storage-minio-devnso.smardigo.digital", + read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ devnso_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ devnso_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ devnso_minio_admin_accesskey }}", + admin_secretkey: "{{ devnso_minio_admin_secretkey }}", + hour: "3", + minute: "30", + bucket: "postgres" + } + - { + stage: "devnso", + url: "https://s3storage-minio-devnso.smardigo.digital", + read_only_accesskey: "{{ devnso_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ devnso_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ devnso_minio_user_read_write_wordpress_accesskey }}", + read_write_secretkey: "{{ devnso_minio_user_read_write_wordpress_secretkey }}", + admin_accesskey: "{{ devnso_minio_admin_accesskey }}", + admin_secretkey: "{{ devnso_minio_admin_secretkey }}", + hour: "3", + minute: "40", + bucket: "wordpress" + } + - { + stage: "keycloak", + url: "https://s3storage-keycloak-devnso.smardigo.digital", + read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ keycloak_minio_admin_accesskey }}", + admin_secretkey: "{{ keycloak_minio_admin_secretkey }}", + hour: "3", + minute: "50", + bucket: "postgres" + } diff --git a/group_vars/stage_devnso/vault_backup.yml b/group_vars/stage_devnso/vault_backup.yml index f15301f..886c0cf 100644 --- a/group_vars/stage_devnso/vault_backup.yml +++ b/group_vars/stage_devnso/vault_backup.yml @@ -1,28 +1,100 @@ $ANSIBLE_VAULT;1.1;AES256 -61356237613639353839316532336566616338666264356562386166306466316337393537363930 -6438313237336536626636623637643330383637643165630a386334303436613837303435306238 -61323237643831353637653234333932393739393239333466656233383361643038366231373631 -3439303035616233640a356431396363383361353561623463656130356130343864363139333765 -33373465383939616662636330343130376235343830643331633531313537663065643439373062 -61303631356563383234633430393663386333346231346631643231313462623835373662383238 -34353737613030643832373832656339333431343336663131343236353131356466323135356563 -38646532303938653535356262653739623333343239346563306161353934633362386239653462 -66306363613632356463643132346134396335343966333539373865363863303635316233636632 -31656433653839396430656637346465343661376534333936323866376362633432646163326665 -34306564613437303634376237356536666435643031313363643265303430336466663865653465 -62643865363165386530363632333935623563396364316332636535376330623163306164396530 -39366661343136636236313566373732653338326132303031356163646665633534623130383339 -63343663313766373664326331663037333631346433333639633936623064613631653234633834 -38646437333231353639336534333166313530336662616131663137643965343266633635326335 -32353938313639336135386462336436363434636665383339386331336463303035613438383834 -61306563373261393833643465633930336465376163353631353166656133323830633632613432 -63633830343230396265633831613535613033643538316230373166656566646563306634363761 -34326430623838303637343333663537613265636261353765623764666636313330663532306336 -66346566653239633965386162666434633662333334386331616235393038636331663438643239 -64623965313236366661396663346433363835656633386463376634383537666632663138343931 -32613232393265633233343134663630656633303531623764623633373964393431313066613834 -35313036666230623263666437306430616139336661383032346132646565616435346166353434 -62636466373662303232326437653536326666666639396237613261636633323765393561336464 -31613930343934313465616631616663316464303064323861343335653737316463613566636536 -36303937643339316164616136393635386436383231393938313534303666636465323666633361 -396239303536383337653033383637376532 +63653638613062393562373465663835323662383037386161613336313433663032623733393638 +6334343336396535373865306237613730613032396439630a343238633830336132653832353734 +39626662313034346630323864326138313335626565353834333337626164653236333236613566 +6333363364363236310a643666366665333265303435383834613332363334646636636666663366 +38663636306232363864356539613933376363333739386634316137306461346231633833343131 +62396262316135643031396135316635633766303936623535323031303636666661366431316233 +31613066313462366662386463363236666430663430643135383132366536636165653636626633 +62633438376361643961373066363836376162316233393038656236346364303530313238333535 +32663065393236336131386239366436303731383836376531643262393463393166663931633066 +31633732613539333238633364383863393963363261356139396137303563386434353932656365 +34653363373665376665616539326635306433383061376332316633623365336366356233666635 +63323739326233326538666232623231653839313561656632636539646137313936323138616165 +66643162383930633637656134353365626134616533333763306265646133366663306330643363 +31336239323638663732383937336263393638616161346132393332353237303534366334613864 +63353534313533396438653761623930326262646161386536623333396664343233373135613133 +36613033643733616265633832613733613730343537343632336435326432343165316331343439 +32656366346665313831353630343831363332336465333364363962326331316363663963373766 +62616562646237653263313561616334383437653936613437303735373238333363316539373465 +37386636633366663832303133343639303237343963306139613134653232616135393238386635 +37363238613962333033376462653562306130366233633035396235333537306234323131373734 +32373430323638616166643438313965643630636533663330373139303237656532663437363536 +36336634363366653435626166323136323439333063306139633838323034323236626635666564 +61306232643337643432373238653935616336613639363964333439623033323339383631313838 +39636331376534656337636663323265343839316339616162373332656438363739323366616634 +63343966326231396562613038306530303762663338366435373034623439383930373033633730 +37373033373065383462343862326338333130323330653633333532316338326363313865316337 +38613633353535316631343931626661616535313133303433383663376138396464633961643435 +63666137396330653461323737636532303064646434326136613666386430343966313733323433 +36636632633635353134626630326235373466393661306333353431666335646333386665613333 +38653831663636393261326639643931363633633835353436373832316665386235366463373438 +61643465633730316334396231633337333533663461343432396631393035383035376338663438 +65323765366335613062353863616237373066346332633864636535393638386236393163666266 +34393163386232633034366637306333363130646136363630376134396539373439306461306333 +63356338646132623031356638313434393638623935633038386662376237666362326134343137 +66393661626562656263303235646530303839383562643437313334356464666532343662303237 +66333139333961663733663462626430633365303031633736393731646239326236353839343162 +66656263613866383666663839366234333830666261303665333662333434333432633662363232 +38363430343863343733626662613931386266333261303263613865636437353233343530336331 +37316535633765346430303439333235653139386438656562613331363663336637656237666539 +64366362646236303738663062646539306430353838653639323331313237303533303932373964 +61616637303562396237636238626139376139643430366162613665363830343736646333396634 +30323435336566343265336166303361623435323037366136633864616532326531306164393532 +38343639393539373839393261613966643239303961323137353361656161326564373035383461 +38653934363262613661346639326232306564303434356635656232343632376237633334613863 +62386630373261306339373065366537333033616264363265303539333132336531393135326262 +38366232326536333435656337363638396133366234336164353737336362643034623161646461 +34623262613333646561666135303061653164633266383030646364313437623834313735373863 +65623834646533343338343132373962623739393830623230663130316239623564383563663533 +33303964353462663734656538356336326437313534383062366564373631353764646262313934 +62663637303565623562373432613664646439346631666235373139366335343065313633643164 +31386639373635616136303636363034363362386134656639363166363136313939316530363666 +61666635656231393965663630373961623631653831393631623634376138643930663466613732 +34333465633762303065333162653366363333313962626563373236303432623164656565383533 +31366530353539383665336265326238393133366538396266656463393739373036313966646136 +36333261613865316431653233626561366531663335306339316538346337386639376161623038 +30303665383064323162393432633530303162333865356337373131623834353230363265313734 +62613263323630666336383839323639383262323938363531323131646363343136353664313663 +63396263353139393037343834333665303135656366663637653662616235376437303238306434 +62323633373063643861373833343762373039666233343866396531326639376632366239613133 +34356266343035323265363534643831373166336364653232353766646262613337653435323330 +30653538316264313933623563386462643832356334393032613465383737616337636131356332 +36313130323066366537356136393431396263306136663038366536336165376462616562616362 +66313132313665663366356230313965373335326332366433633031386139373061643335386231 +63396431393630623432313130316364353835663935663064303639613263366236653261333861 +33616162613335363163393661623766366565343539303734333461306564376139303336303835 +62333362643365326338623262653961353933386563643438366463326132376537346337383461 +39323230366533646630376338353263623335306330326334326233326666616537653164656632 +36626231366132306533323132323134333337303731323133313933376231376662653133653964 +37346139663162313731653261353630313866616630643738373766666564373033643333306665 +36626235623762313362313035623939373435333366323135363034366262336239616537626338 +31653763666663383637663538626236303761343934366635636538663166663261666263303230 +33323565376434323632336532633664316664616564643465653831633562303032393131333531 +36373964313863656663343461313931336431333864343332333038383533353339666532333730 +30356364353662383034313131326431663463656162383533356235666163636262326437383737 +33363739643533333663326132333333356331336230383738646464366333363561616435643830 +62333037366462316564306366366461616136363963616233383531636465373731326438316565 +37313630613836306465306238336435616261343263313739373664343363656264633764393030 +32316431616431333537383037386231363139643564363462316338303461336631363463636330 +32663632346239313634633261613165613433313731613164623165636234393434353132386562 +35353239653638333632656336643338643363613833366163333338343364656461353631666630 +37643535643662653635613761376133366363633936366363353130613132366532643164613864 +64383264306235666663346166613561363461376233336133663962376563663761303434613063 +65623630383430656536326238363632313539306332643231633131326133343731326337623036 +36346665636532316363393339636438613133363937653436656333336465353035393064316665 +62613635376664366363646663633330343034313037633665653565353931633235346438323631 +64626134326666613330643432346633396464613730346438623937613565613030626361663730 +64366334623563656566393865613964613136623562396366383964646538356166393461623530 +34633430626637633636313866316463636431393030336630623733393031616564363161653830 +37383136363631643661356239613431383331383639393064353734653561326262636466343838 +34616330343631356536386634383164616531306264343630666163326261303166613366363235 +64306464306631383633306264626338666439643562373966396332663937663839303965333733 +34343033363665353763613463663363353735613635393263636135376430363062393635363864 +65313666363864666263623634326661636531366435616637303535363731306131343761313663 +30363639386564323965333738383236626334653464373331333062666230353834323062343236 +30663431313537623661633263366435393632383030633631616631363732646431323135643738 +30363561343637393761376130323034323831666535303563313130623664666439383539633234 +39636665633361646537653766333963333636313764383363666665626663353264613565376439 +38373362373239303131343131326333666230393433373734393431353537656334303031663365 +346132616235663836643932323633663662 diff --git a/group_vars/stage_prodwork01/backup.yml b/group_vars/stage_prodwork01/backup.yml index 031cab3..b4c8163 100644 --- a/group_vars/stage_prodwork01/backup.yml +++ b/group_vars/stage_prodwork01/backup.yml @@ -1,41 +1,201 @@ backup_lvm_hcloudvol_size: 30 backup_lvm_hcloudvol_count: 8 -minio_nsodev_accesskey: "{{ minio_nsodev_accesskey_vault }}" -minio_nsodev_secretkey: "{{ minio_nsodev_secretkey_vault }}" +backup_user_ssh_privkey: "{{ backup_user_ssh_privkey_vault }}" -minio_cusqa_accesskey: "{{ minio_cusqa_accesskey_vault }}" -minio_cusqa_secretkey: "{{ minio_cusqa_secretkey_vault }}" +# Admin access for S3 Storage on stage nsodev +nsodev_minio_admin_accesskey: "{{ nsodev_minio_admin_accesskey_vault }}" +nsodev_minio_admin_secretkey: "{{ nsodev_minio_admin_secretkey_vault }}" -minio_cusprod_accesskey: "{{ minio_cusprod_accesskey_vault }}" -minio_cusprod_secretkey: "{{ minio_cusprod_secretkey_vault }}" +# Readonly access for S3 Storage on stage nsodev all buckets +nsodev_minio_user_read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey_vault }}" +nsodev_minio_user_read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage nsodev bucket wordpress +nsodev_minio_user_read_write_wordpress_accesskey: "{{ nsodev_minio_user_read_write_wordpress_accesskey_vault }}" +nsodev_minio_user_read_write_wordpress_secretkey: "{{ nsodev_minio_user_read_write_wordpress_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage nsodev bucket postgres +nsodev_minio_user_read_write_postgres_accesskey: "{{ nsodev_minio_user_read_write_postgres_accesskey_vault }}" +nsodev_minio_user_read_write_postgres_secretkey: "{{ nsodev_minio_user_read_write_postgres_secretkey_vault }}" + + +# Admin access for S3 Storage on stage cusqa +cusqa_minio_admin_accesskey: "{{ cusqa_minio_admin_accesskey_vault }}" +cusqa_minio_admin_secretkey: "{{ cusqa_minio_admin_secretkey_vault }}" + +# Readonly access for S3 Storage on stage cusqa all buckets +cusqa_minio_user_read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey_vault }}" +cusqa_minio_user_read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage cusqa bucket wordpress +cusqa_minio_user_read_write_wordpress_accesskey: "{{ cusqa_minio_user_read_write_wordpress_accesskey_vault }}" +cusqa_minio_user_read_write_wordpress_secretkey: "{{ cusqa_minio_user_read_write_wordpress_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage cusqa bucket postgres +cusqa_minio_user_read_write_postgres_accesskey: "{{ cusqa_minio_user_read_write_postgres_accesskey_vault }}" +cusqa_minio_user_read_write_postgres_secretkey: "{{ cusqa_minio_user_read_write_postgres_secretkey_vault }}" + + +# Admin access for S3 Storage on stage cusprod +cusprod_minio_admin_accesskey: "{{ cusprod_minio_admin_accesskey_vault }}" +cusprod_minio_admin_secretkey: "{{ cusprod_minio_admin_secretkey_vault }}" + +# Readonly access for S3 Storage on stage cusprod all buckets +cusprod_minio_user_read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey_vault }}" +cusprod_minio_user_read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage cusprod bucket wordpress +cusprod_minio_user_read_write_wordpress_accesskey: "{{ cusprod_minio_user_read_write_wordpress_accesskey_vault }}" +cusprod_minio_user_read_write_wordpress_secretkey: "{{ cusprod_minio_user_read_write_wordpress_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage cusprod bucket postgres +cusprod_minio_user_read_write_postgres_accesskey: "{{ cusprod_minio_user_read_write_postgres_accesskey_vault }}" +cusprod_minio_user_read_write_postgres_secretkey: "{{ cusprod_minio_user_read_write_postgres_secretkey_vault }}" + + +# Admin access for S3 Storage on stage keycloak +keycloak_minio_admin_accesskey: "{{ keycloak_minio_admin_accesskey_vault }}" +keycloak_minio_admin_secretkey: "{{ keycloak_minio_admin_secretkey_vault }}" + +# Readonly access for S3 Storage on stage keycloak all buckets +keycloak_minio_user_read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey_vault }}" +keycloak_minio_user_read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey_vault }}" + +# ReadWrite access for S3 Storage on stage keycloak bucket postgres +keycloak_minio_user_read_write_postgres_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey_vault }}" +keycloak_minio_user_read_write_postgres_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey_vault }}" -minio_keycloak_accesskey: "{{ minio_keycloak_accesskey_vault }}" -minio_keycloak_secretkey: "{{ minio_keycloak_secretkey_vault }}" -minio_keycloak_url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital" minio_stage_dicts: - { stage: "nsodev", url: "https://s3storage-nsodev-prodwork01.smardigo.digital", - minio_accesskey: "{{ minio_nsodev_accesskey }}", - minio_secretkey: "{{ minio_nsodev_secretkey }}", + read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ nsodev_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ nsodev_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ nsodev_minio_admin_accesskey }}", + admin_secretkey: "{{ nsodev_minio_admin_secretkey }}", hour: "3", minute: "30", + bucket: "postgres" + } + - { + stage: "nsodev", + url: "https://s3storage-nsodev-prodwork01.smardigo.digital", + read_only_accesskey: "{{ nsodev_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ nsodev_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ nsodev_minio_user_read_write_wordpress_accesskey }}", + read_write_secretkey: "{{ nsodev_minio_user_read_write_wordpress_secretkey }}", + admin_accesskey: "{{ nsodev_minio_admin_accesskey }}", + admin_secretkey: "{{ nsodev_minio_admin_secretkey }}", + hour: "3", + minute: "35", + bucket: "wordpress" } - { stage: "cusqa", url: "https://s3storage-cusqa-prodwork01.smardigo.digital", - minio_accesskey: "{{ minio_cusqa_accesskey }}", - minio_secretkey: "{{ minio_cusqa_secretkey }}", + read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ cusqa_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ cusqa_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ cusqa_minio_admin_accesskey }}", + admin_secretkey: "{{ cusqa_minio_admin_secretkey }}", hour: "3", - minute: "30", + minute: "40", + bucket: "postgres" + } + - { + stage: "cusqa", + url: "https://s3storage-cusqa-prodwork01.smardigo.digital", + read_only_accesskey: "{{ cusqa_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ cusqa_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ cusqa_minio_user_read_write_wordpress_accesskey }}", + read_write_secretkey: "{{ cusqa_minio_user_read_write_wordpress_secretkey }}", + admin_accesskey: "{{ cusqa_minio_admin_accesskey }}", + admin_secretkey: "{{ cusqa_minio_admin_secretkey }}", + hour: "3", + minute: "45", + bucket: "wordpress" + } + - { + stage: "cusprod", + url: "https://s3storage-cusprod-prodwork01.smardigo.digital", + read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ cusprod_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ cusprod_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ cusprod_minio_admin_accesskey }}", + admin_secretkey: "{{ cusprod_minio_admin_secretkey }}", + hour: "3", + minute: "50", + bucket: "postgres" } - { stage: "cusprod", url: "https://s3storage-cusprod-prodwork01.smardigo.digital", - minio_accesskey: "{{ minio_cusprod_accesskey }}", - minio_secretkey: "{{ minio_cusprod_secretkey }}", + read_only_accesskey: "{{ cusprod_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ cusprod_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ cusprod_minio_user_read_write_wordpress_accesskey }}", + read_write_secretkey: "{{ cusprod_minio_user_read_write_wordpress_secretkey }}", + admin_accesskey: "{{ cusprod_minio_admin_accesskey }}", + admin_secretkey: "{{ cusprod_minio_admin_secretkey }}", + hour: "3", + minute: "55", + bucket: "wordpress" + } + - { + stage: "keycloak", + url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital", + read_only_accesskey: "{{ keycloak_minio_user_read_only_accesskey }}", + read_only_secretkey: "{{ keycloak_minio_user_read_only_secretkey }}", + read_write_accesskey: "{{ keycloak_minio_user_read_write_postgres_accesskey }}", + read_write_secretkey: "{{ keycloak_minio_user_read_write_postgres_secretkey }}", + admin_accesskey: "{{ keycloak_minio_admin_accesskey }}", + admin_secretkey: "{{ keycloak_minio_admin_secretkey }}", hour: "4", minute: "0", + bucket: "postgres" } + + +# minio_nsodev_accesskey: "{{ minio_nsodev_accesskey_vault }}" +# minio_nsodev_secretkey: "{{ minio_nsodev_secretkey_vault }}" + +# minio_cusqa_accesskey: "{{ minio_cusqa_accesskey_vault }}" +# minio_cusqa_secretkey: "{{ minio_cusqa_secretkey_vault }}" + +# minio_cusprod_accesskey: "{{ minio_cusprod_accesskey_vault }}" +# minio_cusprod_secretkey: "{{ minio_cusprod_secretkey_vault }}" + +# minio_keycloak_accesskey: "{{ minio_keycloak_accesskey_vault }}" +# minio_keycloak_secretkey: "{{ minio_keycloak_secretkey_vault }}" +# minio_keycloak_url: "https://s3storage-mobene-keycloak-prodwork01.smardigo.digital" + +# minio_stage_dicts: +# - { +# stage: "nsodev", +# url: "https://s3storage-nsodev-prodwork01.smardigo.digital", +# minio_accesskey: "{{ minio_nsodev_accesskey }}", +# minio_secretkey: "{{ minio_nsodev_secretkey }}", +# hour: "3", +# minute: "30", +# } +# - { +# stage: "cusqa", +# url: "https://s3storage-cusqa-prodwork01.smardigo.digital", +# minio_accesskey: "{{ minio_cusqa_accesskey }}", +# minio_secretkey: "{{ minio_cusqa_secretkey }}", +# hour: "3", +# minute: "30", +# } +# - { +# stage: "cusprod", +# url: "https://s3storage-cusprod-prodwork01.smardigo.digital", +# minio_accesskey: "{{ minio_cusprod_accesskey }}", +# minio_secretkey: "{{ minio_cusprod_secretkey }}", +# hour: "4", +# minute: "0", +# } diff --git a/pmci-server-create.yml b/pmci-server-create.yml index dd9a434..91eee22 100644 --- a/pmci-server-create.yml +++ b/pmci-server-create.yml @@ -94,6 +94,7 @@ 'docker-logrotate', 'docker-engine', 'smartmontools', + 'mc', ] state: 'absent' when: ansible_distribution == "Ubuntu" diff --git a/roles/backup_minio/files/keycloak_pull_from_minio_server.sh b/roles/backup/files/keycloak_pull_from_minio_server.sh similarity index 100% rename from roles/backup_minio/files/keycloak_pull_from_minio_server.sh rename to roles/backup/files/keycloak_pull_from_minio_server.sh diff --git a/roles/backup/files/mirror_bucket_from_minio_server.sh b/roles/backup/files/mirror_bucket_from_minio_server.sh new file mode 100644 index 0000000..513c2d8 --- /dev/null +++ b/roles/backup/files/mirror_bucket_from_minio_server.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Script: mirror_bucket_from_minio_server.sh +# This script is run as a cron job. +# It mirrors an S3 bucket to the local backup directory. +# A readonly user is used for S3 access. +# Example: mirror_bucket_from_minio_server.sh https://s3storage-minio-devnso.smardigo.digital devnso minio-readonly-devnso secretkey wordpress +# Version: 1.0 +# Author: ext.Hans-Peter.Wissenbach@netgo.de +# History: 2023.07.27 - rework previous script pull_from_minio_server.sh and add bucket +# 2023.08.09 - add mirror result and error detection + +MINIO_URL=$1 +STAGE=$2 +ACCESSKEY=$3 +SECRETKEY=$4 +BUCKET=$5 + +LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/bucket/${BUCKET}" +METRICS_FILE="${HOME}/metrics_${STAGE}_${BUCKET}.prom" +ALIAS="${STAGE}" + +BACKUP_START=$(date +%s) + +mkdir -p ${LOCAL_BACKUP_DIR} + +rm -rf ${LOCAL_BACKUP_DIR}/* + +mcli alias set ${ALIAS} ${MINIO_URL} ${ACCESSKEY} ${SECRETKEY} +RC="$?" + +if [[ ${RC} -eq "0" ]] +then + RESULT=$(mcli mirror ${ALIAS}/${BUCKET} ${LOCAL_BACKUP_DIR} --overwrite --newer-than 1d --json) + echo "${RESULT}" + TRANSFERRED=$(echo "${RESULT}" | jq '.transferred | select(. != null)') + TRANSFERRED="${TRANSFERRED:=0}" # default 0 if not set + STATUS=$(echo "${RESULT}" | jq -e '.status == "success"') + RC="$?" + + mcli alias rm ${ALIAS} +fi + +BACKUP_END=$(date +%s) + +if [[ ${RC} -eq "0" ]] && [[ ${TRANSFERRED} > 0 ]] +then + echo "Nightly Backup Successful - writing METRICS_FILE: ${METRICS_FILE}" + tee < ${METRICS_FILE} +# HELP nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970). +# TYPE nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} gauge +nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_START} + +# HELP nightly_backup_successful_${STAGE}_${BUCKET} +# TYPE nightly_backup_successful_${STAGE}_${BUCKET} gauge +nightly_backup_successful_${STAGE}_${BUCKET}{stage="${STAGE}"} 0 + +# HELP nightly_backup_transferred_bytes_${STAGE}_${BUCKET} +# TYPE nightly_backup_transferred_bytes_${STAGE}_${BUCKET} gauge +nightly_backup_transferred_bytes_${STAGE}_${BUCKET}{stage="${STAGE}"} ${TRANSFERRED} + +# HELP nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970). +# TYPE nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} gauge +nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_END} +EOF +else + echo "Nightly Backup Failed - writing METRICS_FILE: ${METRICS_FILE}" + tee < ${METRICS_FILE} +# HELP nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970). +# TYPE nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET} gauge +nightly_backup_transfer_started_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_START} + +# HELP nightly_backup_successful_${STAGE}_${BUCKET} +# TYPE nightly_backup_successful_${STAGE}_${BUCKET} gauge +nightly_backup_successful_${STAGE}_${BUCKET}{stage="${STAGE}"} 1 + +# HELP nightly_backup_transferred_bytes_${STAGE}_${BUCKET} +# TYPE nightly_backup_transferred_bytes_${STAGE}_${BUCKET} gauge +nightly_backup_transferred_bytes_${STAGE}_${BUCKET}{stage="${STAGE}"} ${TRANSFERRED} + +# HELP nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} System time in seconds since epoch (1970). +# TYPE nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET} gauge +nightly_backup_transfer_ended_seconds_${STAGE}_${BUCKET}{stage="${STAGE}"} ${BACKUP_END} +EOF +fi + +exit ${RC} diff --git a/roles/backup_minio/files/pull_from_minio_server.sh b/roles/backup/files/pull_from_minio_server.sh similarity index 100% rename from roles/backup_minio/files/pull_from_minio_server.sh rename to roles/backup/files/pull_from_minio_server.sh diff --git a/roles/backup/files/read_only_policy.json b/roles/backup/files/read_only_policy.json new file mode 100644 index 0000000..751ff08 --- /dev/null +++ b/roles/backup/files/read_only_policy.json @@ -0,0 +1,16 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:GetObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::*" + ], + "Sid": "ReadOnlyPolicy" + } + ] +} \ No newline at end of file diff --git a/roles/backup/files/read_write_postgres_policy.json b/roles/backup/files/read_write_postgres_policy.json new file mode 100644 index 0000000..bf3ad65 --- /dev/null +++ b/roles/backup/files/read_write_postgres_policy.json @@ -0,0 +1,18 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::postgres/*", "arn:aws:s3:::postgres" + ], + "Sid": "ReadWritePostgresPolicy" + } + ] +} \ No newline at end of file diff --git a/roles/backup/files/read_write_wordpress_policy.json b/roles/backup/files/read_write_wordpress_policy.json new file mode 100644 index 0000000..dd5972e --- /dev/null +++ b/roles/backup/files/read_write_wordpress_policy.json @@ -0,0 +1,18 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::wordpress/*", "arn:aws:s3:::wordpress" + ], + "Sid": "ReadWriteWordpressPolicy" + } + ] +} \ No newline at end of file diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml index aa66181..5400858 100644 --- a/roles/backup/tasks/main.yml +++ b/roles/backup/tasks/main.yml @@ -41,7 +41,7 @@ group: '{{ system_user }}' content: '{{ backup_user_ssh_privkey_vault }}' -- name: "Providing rsync script" +- name: "Providing Backup scripts" become: yes copy: src: '{{ item }}' @@ -52,6 +52,10 @@ with_items: - pull_remote_backups.sh - push_backups_to_restore_server.sh + - mirror_bucket_from_minio_server.sh + - read_only_policy.json + - read_write_postgres_policy.json + - read_write_wordpress_policy.json - name: Touch metrics.prom if not exists file: @@ -94,3 +98,124 @@ src: "/home/{{ system_user }}/backup_status_postgres.prom" dest: "/var/lib/prometheus/node-exporter/backup_status_postgres.prom" state: link + +- name: Recursively change ownership of backups directory + ansible.builtin.file: + path: /home/{{ system_user }}/backups + state: directory + recurse: yes + owner: '{{ system_user }}' + group: '{{ system_user }}' + +- name: Download minio client + become: yes + ansible.builtin.get_url: + url: https://dl.min.io/client/mc/release/linux-amd64/mc + dest: /usr/bin/mcli + mode: '0755' + +- name: "Set MinIO alias for {{ item.stage }}_admin" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli alias set {{ item.stage }}_admin {{ item.url }} {{ item.admin_accesskey }} {{ item.admin_secretkey }}' + loop: "{{ minio_stage_dicts }}" + +- name: "Add MinIO read only users {{ item.read_only_accesskey }}" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin user add {{ item.stage }}_admin {{ item.read_only_accesskey }} {{ item.read_only_secretkey }}' + loop: "{{ minio_stage_dicts }}" + +- name: "Add MinIO read write user {{ item.read_write_accesskey }}" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin user add {{ item.stage }}_admin {{ item.read_write_accesskey }} {{ item.read_write_secretkey }}' + loop: "{{ minio_stage_dicts }}" + +- name: "Create MinIO read only policy" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin policy create {{ item.stage }}_admin read_only_policy /home/{{ system_user }}/read_only_policy.json' + loop: "{{ minio_stage_dicts }}" + +- name: "Attach MinIO read only policy to user {{ item.read_only_accesskey }}" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin policy attach {{ item.stage }}_admin read_only_policy --user {{ item.read_only_accesskey }}' + loop: "{{ minio_stage_dicts }}" + register: policy_read_only_result + failed_when: "'policy is already attached' not in policy_read_only_result.stderr and policy_read_only_result.rc == 1" + +- name: "Create MinIO read write policy per bucket" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin policy create {{ item.stage }}_admin read_write_{{ item.bucket }}_policy /home/{{ system_user }}/read_write_{{ item.bucket }}_policy.json' + loop: "{{ minio_stage_dicts }}" + +- name: "Attach MinIO read write policy to user {{ item.read_write_accesskey }}" # noqa command-instead-of-shell no-changed-when + become: true + become_user: '{{ system_user }}' + ansible.builtin.shell: 'mcli admin policy attach {{ item.stage }}_admin read_write_{{ item.bucket }}_policy --user {{ item.read_write_accesskey }}' + loop: "{{ minio_stage_dicts }}" + register: policy_read_write_result + failed_when: "'policy is already attached' not in policy_read_write_result.stderr and policy_read_write_result.rc == 1" + +# wird abgelöst durch mirror_bucket_from_minio_server.sh +# - name: Create Cron Job for pull_from_minio_server.sh script +# ansible.builtin.cron: +# name: "pull minio backups for {{ item.stage }}" +# hour: "{{ item.hour }}" +# minute: "{{ item.minute }}" +# user: '{{ system_user }}' +# job: "/home/{{ system_user }}/pull_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.minio_accesskey }} {{ item.minio_secretkey }}" +# loop: "{{ minio_stage_dicts }}" + +# wird abgelöst durch mirror_bucket_from_minio_server.sh +# - name: Create Cron Job for keycloak_pull_from_minio_server.sh script +# ansible.builtin.cron: +# name: "pull minio backups for keycloak" +# hour: "2" +# minute: "30" +# user: '{{ system_user }}' +# job: "/home/{{ system_user }}/keycloak_pull_from_minio_server.sh {{ minio_keycloak_url }} {{ minio_keycloak_accesskey }} {{ minio_keycloak_secretkey }}" + +- name: "Create Cron Job for each bucket with mirror_bucket_from_minio_server.sh script" + ansible.builtin.cron: + name: "pull minio backups for {{ item.stage }} and bucket {{ item.bucket }}" + hour: "{{ item.hour }}" + minute: "{{ item.minute }}" + user: '{{ system_user }}' + job: "/home/{{ system_user }}/mirror_bucket_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.read_only_accesskey }} {{ item.read_only_secretkey }} {{ item.bucket }}" + loop: "{{ minio_stage_dicts }}" + +- name: Touch metrics_{{ item.stage }}_{{ item.bucket }}.prom if not exists + file: + path: "/home/{{ system_user }}/metrics_{{ item.stage }}_{{ item.bucket }}.prom" + state: touch + mode: '0744' + owner: '{{ system_user }}' + group: '{{ system_user }}' + loop: "{{ minio_stage_dicts }}" + +- name: Create symbolic link for node_exporter text {{ item.stage }} metrics + file: + src: "/home/{{ system_user }}/metrics_{{ item.stage }}_{{ item.bucket }}.prom" + dest: "/var/lib/prometheus/node-exporter/metrics_{{ item.stage }}_{{ item.bucket }}.prom" + state: link + loop: "{{ minio_stage_dicts }}" + +# wird nicht mehr benötigt wenn umgestellt auf mirror_bucket_from_minio_server.sh +# - name: Touch metrics_keycloak.prom if not exists +# file: +# path: "/home/{{ system_user }}/metrics_keycloak.prom" +# state: touch +# mode: '0744' +# owner: '{{ system_user }}' +# group: '{{ system_user }}' + +# wird nicht mehr benötigt wenn umgestellt auf mirror_bucket_from_minio_server.sh +# - name: Create symbolic link for node_exporter text nsodev metrics +# file: +# src: "/home/{{ system_user }}/metrics_keycloak.prom" +# dest: "/var/lib/prometheus/node-exporter/metrics_keycloak.prom" +# state: link diff --git a/roles/backup_minio/tasks/main.yml b/roles/backup_minio/tasks/main.yml deleted file mode 100644 index 58c8a4f..0000000 --- a/roles/backup_minio/tasks/main.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- - -- name: "Backup storage server | create system user" - become: yes - ansible.builtin.user: - name: '{{ system_user }}' - comment: "user for backup" - shell: /bin/bash - register: create_user - -- name: "Create .ssh dir and backups dir" - become: yes - file: - path: '/home/{{ system_user }}/{{ item.name }}/' - mode: '{{ item.mode }}' - owner: '{{ system_user }}' - group: '{{ system_user }}' - state: directory - loop: - - name: '.ssh' - mode: '0700' - - name: 'backups' - mode: '0775' - -- name: "Create/Resize LVM for datadir" - include_role: - name: lvm_with_hetzner_volumes - vars: - lvm_with_hetzner_volumes__volprefix: backup_datadir - lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}" - lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}" - lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}" - -- name: Recursively change ownership of backups directory - ansible.builtin.file: - path: /home/{{ system_user }}/backups - state: directory - recurse: yes - owner: '{{ system_user }}' - group: '{{ system_user }}' - -- name: Download minio client - become: yes - ansible.builtin.get_url: - url: https://dl.min.io/client/mc/release/linux-amd64/mc - dest: /usr/bin/mc - mode: '0755' - -# - name: "Providing SSH priv.key" -# no_log: true -# become: yes -# copy: -# dest: '/home/{{ system_user }}/.ssh/id_rsa' -# mode: '0400' -# owner: '{{ system_user }}' -# group: '{{ system_user }}' -# content: '{{ backup_user_ssh_privkey_vault }}' - -- name: "Providing mc client script" - become: yes - copy: - src: '{{ item }}' - dest: '/home/{{ system_user }}/{{ item }}' - mode: '0755' - owner: '{{ system_user }}' - group: '{{ system_user }}' - with_items: - - pull_from_minio_server.sh - - keycloak_pull_from_minio_server.sh - -- name: Create Cron Job for pull_from_minio_server.sh script - ansible.builtin.cron: - name: "pull minio backups for {{ item.stage }}" - hour: "{{ item.hour }}" - minute: "{{ item.minute }}" - user: '{{ system_user }}' - job: "/home/{{ system_user }}/pull_from_minio_server.sh {{ item.url }} {{ item.stage }} {{ item.minio_accesskey }} {{ item.minio_secretkey }}" - loop: "{{ minio_stage_dicts }}" - -- name: Create Cron Job for keycloak_pull_from_minio_server.sh script - ansible.builtin.cron: - name: "pull minio backups for keycloak" - hour: "2" - minute: "30" - user: '{{ system_user }}' - job: "/home/{{ system_user }}/keycloak_pull_from_minio_server.sh {{ minio_keycloak_url }} {{ minio_keycloak_accesskey }} {{ minio_keycloak_secretkey }}" - -- name: Touch metrics_nsodev.prom if not exists - file: - path: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom" - state: touch - mode: '0744' - owner: '{{ system_user }}' - group: '{{ system_user }}' - loop: "{{ minio_stage_dicts }}" - - -- name: Create symbolic link for node_exporter text nsodev metrics - file: - src: "/home/{{ system_user }}/metrics_{{ item.stage }}.prom" - dest: "/var/lib/prometheus/node-exporter/metrics_{{ item.stage }}.prom" - state: link - loop: "{{ minio_stage_dicts }}" - -- name: Touch metrics_keycloak.prom if not exists - file: - path: "/home/{{ system_user }}/metrics_keycloak.prom" - state: touch - mode: '0744' - owner: '{{ system_user }}' - group: '{{ system_user }}' - - -- name: Create symbolic link for node_exporter text nsodev metrics - file: - src: "/home/{{ system_user }}/metrics_keycloak.prom" - dest: "/var/lib/prometheus/node-exporter/metrics_keycloak.prom" - state: link - diff --git a/roles/backup_minio/defaults/main.yml b/roles/backup_old/defaults/main.yml similarity index 100% rename from roles/backup_minio/defaults/main.yml rename to roles/backup_old/defaults/main.yml diff --git a/roles/backup_old/files/pull_remote_backups.sh b/roles/backup_old/files/pull_remote_backups.sh new file mode 100644 index 0000000..8c6bef5 --- /dev/null +++ b/roles/backup_old/files/pull_remote_backups.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# +# + +# Fail fast and be aware of exit codes +set -euo pipefail + +# Define some variables +DATE=$(date +%F) +DATE_TIME=$(date +%F_%H:%M) +REMOTE_SYSTEM_USER=backupuser +DATABASE_SERVER=$1 +STAGE=$2 +DATABASE_ENGINE=$3 +DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER} +METRICS_FILE=${HOME}/backup_status_${DATABASE_ENGINE}.prom +LOG_FILE=${DEST_DIR}/backup_${DATE_TIME}.log + +# Create backup directory ${DEST_DIR} if not exist +mkdir -p ${DEST_DIR} + +# Redirect stderr to stdout and save everything to log file +exec > ${LOG_FILE} 2>&1 + +# Log backup sync start time +echo "----- Start backup Sync - ${DATE_TIME} -----" + +# Remove files oder than 48h in ${DEST_DIR} +find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR" +[ "$?" != "0" ] && exit 1 + +echo "Removing logfiles older than 7d ..." +find $DEST_DIR -type f -mtime +7 -name "backup_*.log" -print0 | xargs -I OLD_FILES -0 rm -rf "OLD_FILES" + +# Start rsync job from ${DATABASE_SERVER} to ${DEST_DIR}/ +rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/ +[ "$?" -eq "0" ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1" + +BACKUP_STATUS_FILE=$(ls -t1 ${DEST_DIR}/${DATE}/backup_finished_${DATE}_* | head -n1) +# Check existence of current ${BACKUP_STATUS_FILE}, which is created by AWX, in case of succesful database backup only. +[ -f ${BACKUP_STATUS_FILE} ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1" + +# Add backup status to Prometheus metrics file +if [ "$NIGHTLY_BACKUP_SUCCESSFUL" -eq "0" ]; then + echo "NIGHTLY_BACKUP_SUCCESSFUL=0 - writing METRICS_FILE" +cat < $METRICS_FILE +# HELP nightly_backup_successful_${DATABASE_ENGINE} +# TYPE nightly_backup_successful_${DATABASE_ENGINE} gauge +nightly_backup_successful_${DATABASE_ENGINE}{stage="$STAGE"} $NIGHTLY_BACKUP_SUCCESSFUL +nightly_backup_successful_${DATABASE_ENGINE}_finished_seconds{stage="$STAGE"} `date +%s` +EOF + +else + echo "NIGHTLY_BACKUP_SUCCESSFUL=1 - removing METRICS_FILE to trigger alert" + rm $METRICS_FILE +fi + +# Log backup sync end time +echo "----- End backup Sync - ${DATE_TIME} -----" diff --git a/roles/backup_old/files/push_backups_to_restore_server.sh b/roles/backup_old/files/push_backups_to_restore_server.sh new file mode 100644 index 0000000..e654768 --- /dev/null +++ b/roles/backup_old/files/push_backups_to_restore_server.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# +# + +REMOTE_SYSTEM_USER=backupuser +RESTORE_SERVER=$1 +DATABASE_SERVER=$2 +STAGE=$3 +DATABASE_ENGINE=$4 + +# currently it defaults to todays date +DATE=$(date +%F) + +LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}" +BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1) + +REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}/${DATABASE_SERVER}" +DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/" + +if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then + echo "BACKUP_FILE_FOR_TRANSFER not found. EXIT" && exit 1 +fi + +# avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process +ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${RESTORE_SERVER} + +SSH_OPTIONS='-o StrictHostKeyChecking=no' + +# needed due to unknown rsync option --mkpath in rsync version 3.1.3 +ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "mkdir -p ${DEST_DIR}" + +rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER}:${DEST_DIR} + +BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}') + +ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${RESTORE_SERVER} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}" + diff --git a/roles/backup_old/tasks/main.yml b/roles/backup_old/tasks/main.yml new file mode 100644 index 0000000..f0f6f3c --- /dev/null +++ b/roles/backup_old/tasks/main.yml @@ -0,0 +1,96 @@ +--- + +- name: "Backup storage server | create system user" + become: yes + ansible.builtin.user: + name: '{{ system_user }}' + comment: "user for backup" + shell: /bin/bash + register: create_user + +- name: "Create .ssh dir and backups dir" + become: yes + file: + path: '/home/{{ system_user }}/{{ item.name }}/' + mode: '{{ item.mode }}' + owner: '{{ system_user }}' + group: '{{ system_user }}' + state: directory + loop: + - name: '.ssh' + mode: '0700' + - name: 'backups' + mode: '0775' + +- name: "Create/Resize LVM for datadir" + include_role: + name: lvm_with_hetzner_volumes + vars: + lvm_with_hetzner_volumes__volprefix: backup_datadir + lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}" + lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}" + lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}" + +- name: "Providing SSH priv.key" + no_log: true + become: yes + copy: + dest: '/home/{{ system_user }}/.ssh/id_rsa' + mode: '0400' + owner: '{{ system_user }}' + group: '{{ system_user }}' + content: '{{ backup_user_ssh_privkey_vault }}' + +- name: "Providing Backup scripts" + become: yes + copy: + src: '{{ item }}' + dest: '/home/{{ system_user }}/{{ item }}' + mode: '0755' + owner: '{{ system_user }}' + group: '{{ system_user }}' + with_items: + - pull_remote_backups.sh + - push_backups_to_restore_server.sh + +- name: Touch metrics.prom if not exists + file: + path: "/home/{{ system_user }}/metrics.prom" + state: touch + mode: '0744' + owner: '{{ system_user }}' + group: '{{ system_user }}' + +- name: Touch backup_status_maria.prom if not exists + file: + path: "/home/{{ system_user }}/backup_status_maria.prom" + state: touch + mode: '0744' + owner: '{{ system_user }}' + group: '{{ system_user }}' + +- name: Touch backup_status_postgres.prom if not exists + file: + path: "/home/{{ system_user }}/backup_status_postgres.prom" + state: touch + mode: '0744' + owner: '{{ system_user }}' + group: '{{ system_user }}' + +- name: Create symbolic link for node_exporter text metrics + file: + src: "/home/{{ system_user }}/metrics.prom" + dest: "/var/lib/prometheus/node-exporter/offsite-metrics.prom" + state: link + +- name: Create symbolic link for node_exporter text metrics backup_status_maria + file: + src: "/home/{{ system_user }}/backup_status_maria.prom" + dest: "/var/lib/prometheus/node-exporter/backup_status_maria.prom" + state: link + +- name: Create symbolic link for node_exporter text metrics backup_status_postgres + file: + src: "/home/{{ system_user }}/backup_status_postgres.prom" + dest: "/var/lib/prometheus/node-exporter/backup_status_postgres.prom" + state: link diff --git a/setup.yml b/setup.yml index 99e8b45..baf5d5c 100644 --- a/setup.yml +++ b/setup.yml @@ -37,6 +37,7 @@ 'docker-logrotate', 'docker-engine', 'smartmontools', + 'mc', ] state: 'absent' when: ansible_distribution == "Ubuntu" diff --git a/smardigo.yml b/smardigo.yml index 32a878a..0712aa6 100644 --- a/smardigo.yml +++ b/smardigo.yml @@ -67,9 +67,6 @@ - role: backup when: "'backup' in group_names" - - role: backup_minio - when: "'backup_minio' in group_names" - - role: keycloak_compact when: "'keycloak_compact' in group_names" diff --git a/stage-prodwork01 b/stage-prodwork01 index e1a8ed6..f83f533 100644 --- a/stage-prodwork01 +++ b/stage-prodwork01 @@ -1,7 +1,4 @@ -[postfix] -prodwork01-mail-01 - -[backup_minio] +[backup] prodwork01-backup-01 [kube_control_plane] @@ -30,7 +27,7 @@ kube_node [stage_prodwork01:children] postfix k8s_cluster -backup_minio +backup [all:children] stage_prodwork01