From 8318792f1fd4918f44791ac24790d9b47243fee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6rz=2C=20Friedrich?= Date: Thu, 8 Jun 2023 13:01:40 +0000 Subject: [PATCH] DEV-1020 Initial commit new feature branch --- group_vars/all/plain.yml | 10 + group_vars/postgres/plain.yml | 9 +- group_vars/postgres01/plain.yml | 16 + group_vars/postgres02/plain.yml | 18 + group_vars/restore/plain.yml | 5 +- group_vars/stage_dev/plain.yml | 7 + group_vars/stage_dev/vault.yml | 1180 +++++++++-------- host_vars/dev-postgres02-01.yml | 3 + host_vars/dev-postgres02-02.yml | 3 + patchday.yml | 65 +- .../tasks/_remove_hetzner_volumes.yml | 19 +- roles/lvm_with_hetzner_volumes/tasks/main.yml | 64 +- roles/postgres/tasks/base-requirements.yml | 33 +- roles/postgres/tasks/main.yml | 5 +- roles/postgres/tasks/master-requirements.yml | 188 ++- roles/postgres/tasks/slave-requirements.yml | 11 +- stage-dev | 10 +- 17 files changed, 905 insertions(+), 741 deletions(-) create mode 100644 group_vars/postgres01/plain.yml create mode 100644 group_vars/postgres02/plain.yml create mode 100644 host_vars/dev-postgres02-01.yml create mode 100644 host_vars/dev-postgres02-02.yml diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index d563224..9f72f9f 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -145,12 +145,15 @@ docker_compose_path: "/usr/bin/docker-compose" service_base_path: "/etc/smardigo" devops_email_address: "nso.devops@netgo.de" +gitea_admin_email: "{{ devops_email_address }}" lets_encrypt_email: "{{ devops_email_address }}" connect_admin_email: "{{ devops_email_address }}" keycloak_admin_email: "{{ devops_email_address }}" pgadmin4_admin_email: "{{ devops_email_address }}" grafana_admin_email: "{{ devops_email_address }}" grafana_smardigo_email: "{{ devops_email_address }}" +harbor_oidc_admin_email: "{{ devops_email_address }}" +argocd_admin_email: "{{ devops_email_address }}" http_port: "80" https_port: "443" @@ -206,6 +209,12 @@ upstream_dns_servers: - 185.12.64.1 - 185.12.64.2 +harbor_username: "{{ docker_registry_username_vault }}" +harbor_token: "{{ docker_registry_token_vault }}" + +keycloak_admin_username: "keycloak-admin" +keycloak_admin_password: "{{ keycloak_admin_password_vault }}" + # Note: all dollar signs in the hash need to be doubled for escaping. # To create user:password pair, it's possible to use this command: # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g @@ -214,3 +223,4 @@ traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}" mysql_root_username: "{{ mysql_root_username_vault }}" mysql_root_password: "{{ mysql_root_password_vault }}" +lvm_volume_encryption: false diff --git a/group_vars/postgres/plain.yml b/group_vars/postgres/plain.yml index 6a5cf2d..5683d32 100644 --- a/group_vars/postgres/plain.yml +++ b/group_vars/postgres/plain.yml @@ -1,5 +1,4 @@ --- - hetzner_server_type: cpx11 hetzner_server_labels: "stage={{ stage }} service=postgres role={{ server_type }}" @@ -10,4 +9,10 @@ traefik_enabled: false filebeat_postgres_enabled: true custom_plattform_users: - - '{{ backupuser_user_name }}' + - "{{ backupuser_user_name }}" + +postgres_homedir: "/var/lib/postgresql" + +postgres_pgdatadir_lvm_hcloudvol_size: 10 +postgres_pgdatadir_lvm_hcloudvol_count: 1 +postgres_pgdatadir_lvm_hcloudvol_mountpath: "{{ postgres_homedir }}" diff --git a/group_vars/postgres01/plain.yml b/group_vars/postgres01/plain.yml new file mode 100644 index 0000000..29c3ba9 --- /dev/null +++ b/group_vars/postgres01/plain.yml @@ -0,0 +1,16 @@ +--- +shared_service_pg_master_ip: "{{ stage_server_infos + | selectattr('name', 'match', stage + '-postgres-01' ) + | map(attribute='private_ip') + | list + | first + | default('-') }}" +shared_service_pg_slave_ip: "{{ stage_server_infos + | selectattr('name', 'match', stage + '-postgres-02' ) + | map(attribute='private_ip') + | list + | first + | default('-') }}" + +shared_service_postgres_01_hostname: "{{ stage }}-postgres-01" +shared_service_postgres_02_hostname: "{{ stage }}-postgres-02" diff --git a/group_vars/postgres02/plain.yml b/group_vars/postgres02/plain.yml new file mode 100644 index 0000000..3d594a0 --- /dev/null +++ b/group_vars/postgres02/plain.yml @@ -0,0 +1,18 @@ +--- +shared_service_pg_master_ip: "{{ stage_server_infos + | selectattr('name', 'match', stage + '-postgres02-01' ) + | map(attribute='private_ip') + | list + | first + | default('-') }}" +shared_service_pg_slave_ip: "{{ stage_server_infos + | selectattr('name', 'match', stage + '-postgres02-02' ) + | map(attribute='private_ip') + | list + | first + | default('-') }}" + +shared_service_postgres_01_hostname: "{{ stage }}-postgres02-01" +shared_service_postgres_02_hostname: "{{ stage }}-postgres02-02" + +lvm_volume_encryption: true diff --git a/group_vars/restore/plain.yml b/group_vars/restore/plain.yml index 61bc090..66e604d 100644 --- a/group_vars/restore/plain.yml +++ b/group_vars/restore/plain.yml @@ -1,5 +1,4 @@ --- - hetzner_server_type: "{{ hetzner_server_type_restore_database | default('cpx21') }}" hetzner_server_labels: "stage={{ stage }} service=restore database_engine={{ database_engine | default('') }}" @@ -9,8 +8,8 @@ filebeat_enabled: false node_exporter_enabled: false custom_plattform_users: - - '{{ backupuser_user_name }}' + - "{{ backupuser_user_name }}" # postgresql related -# defining type of server (naster|slave|restore) +# defining type of server (master|slave|restore) server_type: restore diff --git a/group_vars/stage_dev/plain.yml b/group_vars/stage_dev/plain.yml index cba340a..023a588 100644 --- a/group_vars/stage_dev/plain.yml +++ b/group_vars/stage_dev/plain.yml @@ -52,6 +52,13 @@ alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_v netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}" netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}" +management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}" + +# smardigo automation {{ stage }} gpg key +# https://git.dev-at.de/smardigo-hetzner/communication-keys/ +# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/ +gpg_key_smardigo_automation__private: "{{ gpg_key_smardigo_automation__private__vault }}" + custom_stage_plattform_users: - hp.wissenbach diff --git a/group_vars/stage_dev/vault.yml b/group_vars/stage_dev/vault.yml index 123958d..3d86df9 100644 --- a/group_vars/stage_dev/vault.yml +++ b/group_vars/stage_dev/vault.yml @@ -1,589 +1,593 @@ $ANSIBLE_VAULT;1.1;AES256 -66613632316632383066323066303432623962633133633130633832393736373731343137303766 -3632303132393266646138396530636334303934633663630a653462626237613432323630383236 -31323033643833623239303161373263616334303165653934373034646232353562366338396438 -6336663965623066380a663232346434333566396233393834653630326465646634393633633162 -30373439346636623934313337626364326439356432336132336533303236343737353530633039 -64616336313235373162323862346164396163626631663462653265636437356433393432396136 -33303635653731653066623636383131356331366230363331323234336234663837383539326630 -39306430333331636330616461383362386463373265626161666139616366393864393136326439 -64656633343138633235653862346361316234383634613164396238303635323961383038323665 -62373037633436656139623533376137623439643137343231646235313338666334636437373336 -37313632346131666165626566363566653637333966376562333936333836636334653166646464 -64646433623332616634326165306364656430343364363239323963393964353864313237663662 -65326565646565633930373765303732386534383530633830646362653438643435383533343864 -61663565343238663133376665643435333765383936623933363862386639383863656535313639 -32646132616335363235366261393835616466303662383639313163653166646163326665356134 -64343361376164323033303861316662656538343933356364343831316339376466313038303762 -64383364653739333762346435356235633238313963663764303433653965623363376236333737 -30626533373837373636373661396332316465616231666536613363366463646339306231343265 -36363335363431373231316263666266333538383833333334343539366236376338323531626533 -37313164653131326431623961316533376138653131613766396236366366323665333437613834 -35623066643637633164336637663635653238396166363738306561616465363265616634656637 -33303538613564646637666136626339653938626661626630613831346161363763663531653266 -62383364626366393232323032353134343465323665313037363064333066396130613533353965 -62383133626363623365333464303336613038373839613561393533366663313535306464633438 -61303165396530663431363238656537616636306337623530383232353863323962663230393032 -38636233626636373535626633383932303932646638376633323239663764633837333265346132 -62663634373038353965383133323934363166383435373462386236363630303564386531633038 -38353161356234646332363862393236356131313733663634626536626339333463333132653438 -31306332333961633566666238323437313632616536616532303537356466376337303737383431 -30656630646133636164346139393263623165646664383038356531373639333538396332316239 -38653235323436303835383437366462356232313834373562653139663363333138346263373237 -36616532636338343662636162363632313062656535653361313463373431643239666136613965 -33316331633962326437633437336666356662663366653337393263663938353431666564323738 -31373063336335356562396533323962623438623530663166313366313433336565653138313262 -36623130326434616335363266373534636136653938366566336361643537316437373363653566 -36303938656461636162383530643663666233323535656432303735306339636330303364366437 -37366636643337653662313164336331373433623530306231313531623162323931613935396630 -35336435343263323832663332636265643033643836633730623237306632643233333638656431 -61313662333438656464326565316166323737636431623836366335383964633630666362363534 -34643565343564386631336538383537393334363965303038366332353637313930373538346635 -61346433373934306163643337356261663462326131323536313562353163653331616533633663 -61346365306365653937613536616165363939643565396335323462343930613563383661626265 -66626336333936613232323635646362633665356135633435323363313339653430653732346630 -38386531356363663634333135656239633839623434643661616466666438386162376330356435 -32343431303536653833313539356165353866663362616436653433666431626630666433616439 -64356461313163353136316336386537666635303832306233626666336362643931613334363765 -66373066663363326565383230336136396432336462353065313736643234323665356436613466 -31386531613262666539316466646630393138373233323263386265663239316433623437666136 -63336461383634616262626235653034316561373231663763336131643830616238656330333465 -30613939386261333035386637383465643038383437373462323933613838643633303365636536 -34333138623738393261626132613462613465313564643765636335363862396636626561633738 -31356638386138323364356262613336646639613562353537376636666437343766363866643365 -34326333626633646632636132643936316334643539623264346431393535663337323639366436 -66623433386564393362663335616636366230326665346330633031346339356336396534343365 -37633039646236333965656237353163363136626361373237306365653130653031363831376663 -33303935376634313430643961633265313730376236323565636631326138643230643561623435 -35373262373036313237626536313431653863646563343363306665386432323266356338636263 -66333061383164376537356139663237393361313638303262323037353766383265363233326462 -34376131366265613832333161346465666462303933316633303736343563666664636436323433 -63343534343163373365383436613831626133306666646463313433363935663063336466653964 -34303936653038623338313639343134636637346266623239646337333561633665396631616466 -65306537383461363330306630653836656231316533333165383162323438643832623036653361 -35386261663865336538636431346430323363343733376636626261326466646136653230376439 -64656662386332386639666236663934353031366236346533336137613234633338643633626330 -31363435356537323636346466316366656238326661346139303964306262656332323631336333 -65393266613333373038336365333730613339333036376166363064616562373936663936383030 -33653362623862643563633132363538373536383639313663653233396266333830653132393234 -65613464373538623661356662386463346565666664666565636436346130346536656666653264 -61666631313866663639353365623735393139373763326536343037303339386433643561366236 -31646664333931373932393434323335386139663663306330343834643339643433653666316433 -30353362383735353931326533643666643634333333666237346138623164366539386238366536 -37326161353434656632343966646163396434666238346537393662623036386231396134306136 -30626465623564663938386234643534383933313134346266323563646363666435333635373630 -39373763396435316236356565336635663462376536656239336463336466663333373530336333 -65306235373166623434353436636432366663653833623063333530396166363937346161656134 -62613634663135323237376464356238363239313266616261333237386636356639383137643166 -31383430353435666561303636613763323530653465333934353438653134626633663732623436 -63353436396663323335313837346131373731643930366537653135623637643136643638343938 -33656539623661333863623730363539313065653338306362353336663761663861303936613863 -31313663376631393534623965666161376365346166623233643336303066653939303734313533 -65396534343339306133313266386662613038663064376161643031316237626165643134373763 -34323537636161396665343731663662653235666635623464623731306539313932323534336134 -32643864373761356531626566633365396637623334636263323332633237353337343866343965 -38663635636463343062636231333037663634623166323464366161626431323832663439623130 -64383364646664333563663231623763353939376133613133353234363761353039346634616366 -30333839303262643164313530346330613335333934616262663038366230376264366265666363 -64323366663238646262343465356634386537343262396161306139313234623939653633366561 -66613663623339356237396430666138646161313939306564396363346563393432376236646362 -35323332383465613562613061646262353335393865386131323333653833306164396338663938 -33336266363462626131343663653537633631616330306336626530343937613539393837373532 -62623737363032316530653435633938396463313263623238663034336238633763383865636430 -64656236366361646337626230653465343665343834333534346334306162336665396230346531 -61356563346165383061363932663535306139363532383564303230363864333836383562353866 -35316534303233336337383234663336613261656330363237656161386633613166643632623662 -31383835383435323966306630386464386464616534376238653238346331393466313231396463 -32393838336161336438663939353336356666383435626637376334323337323466393039343563 -64373566363063343762663063306238343732613833663331333733393962316361343735353035 -39333033313263386339653330363661376534343936303661653565653730316532343038313639 -39653533646533623062623036303133313631346166613437343765373131666561653661353436 -33663930613538366333336131383563323139393631616564633434303834323334363366393662 -63306434643236303364353837353537393332326634353531393365613365346636653437626635 -32383061333439373332326164623035373737313361373033343332613262303036313030663162 -63623763343733383062653637373934303339643561653230633363343534336339393038626561 -62623831316263646134363437626363656161336132386461376538646665376438363736376133 -63343634666430386137383964363331343737616662343335663634356264663062353264643033 -36393235316663316661346537303961383162333230396138393732613339613832653734616237 -30613563663462383738393539356165396137333665366130353434666337353330656338646264 -33623735326239663866643464613362663138303262376630346230303836383039633230306361 -37363865656233646664613666613231396363623464643431353965646266363532303931333735 -39316134653037396463663866316637333138646437303864366132636137613664376537353735 -30333634636238336530396662316466653735373739653437643137333564643637303062303832 -31643565363832646138386661326165356264393261343830353263373462643361373561386166 -61653637333763373032303030633331373065373936336639643030316562623930393161336630 -39636532663934346236666166333461663439666132616637663938343834383063386131623061 -38373963373732373036643537663966393637643361353037623631356565343831393663636262 -65303530363764623035633065376162653638316666653335396538616638323639363439373831 -63303937376563393461636235623534333135396436666630323736633531323139633135633362 -31323337643838633337313361643338623663333561646265316430643531386364643231383938 -38343031636263373666336131653633373765363739653963313534626264363037356661393830 -33303935616133353066323836613265613737643130383036343633323265333364356266343735 -65663636373263326464626636336634663831356361346435616336653566333864623365663732 -64663664623030643365393630383835643766316361653865663961386164616365356535313266 -65363365613836663437326463633035386538653235333564666336396138363166316661646662 -32393935333663303834633234373533313936613235396338663632336361633433333731663564 -31613339383366396130653733393235626136626635666533343364623930643534386134373965 -36656265393966306165333539653631316339306131366161643037613732376234353330356336 -65323334356634343230643337326361333930366361626636313363633339313334383663396534 -31386130646137373139623031396337303065666362396439623061323034616239396432343933 -63303439326439626336666538626238326133383230643861386161393461393238316132313063 -34653834636131643737653039343065633038616165343238326638643339303338656534326564 -34373361626630636339363363316230333466333136306632616464663866333037386438306235 -38663931663061336436333361303562346532346230653935666164343539666434613866333130 -66666266393330373835313530646664346637373335366166393030316432383937613430653164 -30623934666661323863656639353138316662393436643664306639646531363462636231656464 -66333764323436646464613364303366333564396463623164383531303930303730646539356232 -39373662316133393839353739353634306132313431393239616638303366653730386531323662 -33616266613233396632383036623136303235356364313163363462363331663963343430336261 -65323330336137393638313635383033633663333966363237316132383165646365616339623636 -66666435373733663064633437343464343933336461396232376631336464333732373133383034 -66313362643039323932303230353030626534303463626236363031366639396338623639323163 -63356166346534363332633366346231356233356261643264626636336635626365336261616538 -34383439343234356563333538616166393461353062663963313031656365373032383935613962 -31336565333433383232653932393933643538343236626331383266316465366330373735356635 -33383636303036303864323833376330383565336565326439623638386534643731643930363837 -65623761653737363533303636663531626366306661656533313863366466373034666535333963 -34663862306133336262646162633063306533646666353832366433616538303566323661613737 -62616465333530613633616437623231313831326530633331663434633633343638643834643963 -30343263363837626561383633323538323530336537353033376366316236356331306130646235 -33383930353532626431346536653739313439356130656162326536383031616638663335633032 -62313236616533366165376632653935656535633939333334356161333562643466613066616139 -34376537393935313763326239346333366339656137386433643664363462633430376535316130 -62343139386530373234373338326632336438353130306633633562323037376361636464313762 -33633038366663633035666162623032316237616437346662636665643937626536643763306338 -37386361393864356137306465393466393766333839656539316237306631326263356165666264 -39383065656461643966363061633638626361666337313233626362313231393364653363303939 -35653637333963613964343833666264383838313765336635346134383166323066373335613762 -36363061653330643939386264656166373432393533656339353636333266333037346133393864 -37363033366237346336653830336330333566306364383065376364643965303333353032346431 -35643631373738613231303933323365343037363838633131653233363662383435373666313437 -66316637396264376634633834396338326264663839386661396638636338363664383137623737 -34636438633564663661323237326530643534373635646433343663663431653939646263643466 -32343661656437383731373637356532643866333835646132666361346665353264366637626533 -34623632396166303866626537313037373536623332613632656664363836313338616663663235 -30326534313264323932653336333662633133383134396165303232663536613264356233633762 -35333664346130323931666166323539343737353634663061646165613364396162383963396266 -63633764613563383432663334333833623032633363363364623332303731313162666235656162 -39633332303036336266366634326237613231333131613462356166626263346263333430303565 -66343233623036306333633033346164653530363431313239396235336138333836633466653565 -63643233643831323566366562386163303163633836316532633137343434376231663332336538 -30343734386464623132366337376634356237666163323635613234623135356432656631656630 -31393262643839633535663731353939323964633834666334353763363033623332616566303237 -62336163646531326635666533616634333438626339373233306263663934313732613965633131 -31613762336363623762313863633838333139333364653761383363346331383839393161663233 -34383662323361626161653331306535383632316261663432333132616562336632653731393730 -39383932633830373737343263633130626431633738396134326637393962366635376535653132 -39623361623161366439376135633766333962613933383164346535363564333433613363663064 -39383166366664303639666266386162343431336139313933626530316632646534626538323535 -64353938616164396636656130386663626563613963626432363965323031623864356363333634 -33356339653064393833626132666430623734396136616539313532376532376466656232646238 -65643661313130323965663134316434643638653836626131386536346462313734373962623239 -66366637363530616465336465303261393532656166313135653762393335373531333939623665 -36623665656438643935643639623561373937653462613533323132653935653333666431333564 -31333731393866633838343662356462386539303763616364333862316264303738623032613762 -36633635346563333733386666613034366466336137313934353433646535313634356632653264 -31393830326534613936626338343133633136383934643535633463623562636561396439376266 -30366565366330373532633265633039323236333163373637363837646366663035343335383135 -66663030663162386164336237656663353132613762323362316534363232376338373538643939 -37343431376337306532323332623535643537316331343136393930633134643830346361653430 -62613866613164656636623262383636663530323663363636643862323234353632323538663934 -34623437303432353266376532316630366434643866386162626161373931633037336566386531 -30333530343939396466363637653237373565393330663837613231646132323738303735656566 -61353930643861373363396439303833653930356537666539326237653131303061333038636434 -35373761326635333331636136333437643161373061393864303161623336396263313030343265 -33653365363465386138396663626264306636363462343335333961666239383632383630306231 -37653038343430653037333333646666643162653638396531316662323031393631396230383464 -34326632306630636336666238386333666266393665303936363132393231346635643833376364 -63353239616364373733633765303764636530366662376261376338613137333862386663623137 -39613339363732303136316361323732316361326438386564303835386134623439653236353830 -62633763646634316561316634323362366537313436323761643363633461386232336634313831 -38343735616331346366366565356638333462633961643834383834613963303764326536633235 -35373931626261623931623332376330313438346437636466633266623230613964356637373265 -34373737343330373066383138376565393934306665383135646362313561326566313565613462 -30666436656636303361666366363761376666646131303962636233316231626561626235646365 -62653331616432366362343864653937353232646237636531656466323964366162353263343130 -65666162383438393236386462323961643366303330623531313937393666326432623735363561 -33356635326434376432363933346630643836636535623566373364373163663635663639386436 -30366630396233393137383331373838663730356133636134376265623730323230373939383333 -31353234343938613334386566663963386266646262303538336361316339316232353765353037 -39353861646334303766613336313534383531303161653763643833376666373335613232633734 -61383637313636623065376331366334313462383233323833343839653364383336393036393136 -62313462343063323430313361376230383539386362326539663036303231363336376661373939 -36353737336236383538383365343030303634666631396137373962306561663131633339356632 -35306265383064613965643136343832353930626264663335383732633464363032646632613961 -37326638626661376262616161616431343263326566383532353365316235346535333235323331 -32346362346662643631396431336132336136336534643161653862653565613634613664366536 -34613739613232353039373262396263343138306361336338626265656162333335656162613832 -32356462373633313135613065663563396233323538643162366465363361663036313437336565 -32616438303661616134656666393263346135646663393737626232656638383964653733643264 -31633637316331663432356137366165383331653565346635376639646464303237663362386437 -37346466366534313039356237323664313166336261663836363939373231633165343863313739 -30653836333538613763653838643035386165353335366230653662333038313361303336353136 -65303765353738316533353936663966326637653466663438316432326465326334363635663735 -31613636306563616133653930353930313361663361343130616433663364613264363036663362 -30346532336632626236373161366434306464666137336538396435646431323037323766633536 -61316162616363356238616661626431633131333735646632613936363661316638653566643130 -65343733373435646136313730653034353461376138646661363938383239303139323635393533 -33353038396436316634323339313632393666343264306363303339663930373132333461623664 -32363731613965646430303237643837366132643637616437363062323139313139393534666565 -62643534663830313333316232626537333935623038626338306164326162616331303437363837 -30653563626436323466626339666165326430343138363463653338383830623961613431343431 -61666166623037303663323261663561333463346261663362336134326335356331663634616238 -65306331386632613838363032633637383261366532343563323461666535343266373735353336 -37303536316363303364356630376330396234326339363932633537353530353633643262373134 -62663365303336303532376336343031343830366635653036313032633566383061636137626232 -39313161346535383866613961326538356666356266343132316134333434653134383061323630 -34323133393962623466656436626263373162326363303533616265333338613031393336386331 -66663034356563323765393831326230636166346638616431386131323639343632353833643762 -32356630623331646432643764373931393562343832353262623933623332323561663035623336 -30636533303233323831376465323966373133616264343432386165613232313465623930643035 -37386430633539393239343334653337393966386132303462346361666462363362306630373539 -63353831373763393036333064346433393339663765646135303564356132623938626539316130 -34346539336137323437623733323063323139356232613131313134646265393236316530373362 -30386136323861316636613532393261313964356435653265366561303938346438336333376637 -33356535306337396431656631616431613837323566383931376166313463623331326630343432 -33343265396438656665616630306332663066343836653833396336626232343764613934346232 -33373130336237616366626530303934393133343331313838653461623830663534656239353266 -34353364633334303134633463373032636364326365303436613632653933373530636535373562 -38326435386565663032326330653761336665343462383036623434633134386632393130316436 -62646637373066313763316538393437313536313563373663306139346264386432653536323438 -34336663323564393535366362313564633337336234623763373536326462626361303237393761 -33633264343930626431313764346664303039643730623036356366393066613234343830333438 -32316265366232366565386366353037343034393134383663313766303239623661666566313635 -65323063313937613330656235396134643666653839613239613232653439383264656430623139 -61396638626261393761343038346639316437663762396334303839343965613863326261623332 -61616633346435393133363564336566383039313034323037313563636362633836383238326637 -35383539653462613537373431363464306463353032313038333933643536323037393263303031 -63333932303166313563396166363330383032303634343238633437663336626664666534396331 -62363266663763633833326164313063343361376131663136343735343232303336383663363863 -66303733613233613262623330653733323336643333313466666337616266613330643231653939 -35333161643663636432363666663333363837636537663066373938623963653861373034376331 -39663532313630636166623465646163386538333533373339373139313535303865383034646261 -62306566646135343634366166356630353736333234643936363534396434626362643464663039 -61363434623531343534336536623830663631373365313766353966656538623231663836306463 -35633037326361383463306435343031306563623031663632383665316132373031363165376234 -66313436663630306438623366616664616166633536306162373934623234386234653132646362 -31393538363963626139313432636230313231646364653535306237666437373161353733353862 -65636437376462383535636566666364363634366530663833373135376564326639653336383436 -39613665313366306239316432343666356633303162393630663635633362633365346137323733 -38326563336533373130343632623366343739373935636430363838643764633065366364643264 -66633238303430663532613764633565386634323461303264336566363636343034633138343834 -34636131346638653337633466636262303438383366663863643731376532623936393437393230 -33373365326339366361326262376331363662633830643065663939376331306436303566316133 -39666437313966373865613165613465636366633439636335373331343932303166353430643039 -38393232636661326530326366623365613962333766663931626362376435656338646539366364 -30363439623561316539306266666162363132383965616336623337666565373862623666613235 -34666232356162613238653462386566646664343333646364653935333936306263636563636439 -36666332616366653262356430653161636435363238363337303436323633383536656237613036 -36386466366530323635316538336237343838623463623062366433656338653065666563663932 -61613464323938333036613561656463623738666664613631346534323036623631343063323665 -34306633653335646237653438373237396663383966366565643563383565383331313461373364 -37633731633565633766303063623232313264396533666130366234393066633563363261396231 -64323537643530613364313331353934383632663263393361353937346561373239653263653464 -63363064623239393231313562346437303935663130376336343532303563643436393834626163 -32306431323238633530633566626138633030326661373638616661363631346433333137346463 -62653534353361653637643132623036613939356431393439653763616266386637333266643439 -39373232623533366666396139663735346330323338636531663139333665326439316335333630 -39633437386136376662383662643439326436643961376638626561363362613061353862373838 -34663762663863313364373439303736373635613830373032376332646137323634326262353263 -37633366626632646264306532396335396165653565666435343461303930346638353138333639 -30306637386131316361313633333432383030373534323635376663316536366538343835313632 -30383532346236393664393930343238343134386330386663373861633534313265393435393635 -34646333386233663461353537333363333765383838303539386231323434396633313031356362 -66623765343865386335666261323266326631343166633430383131616638376461393238666333 -36666661386162343930653636313836376639333939623566646532613835656132383063393034 -37643632383838653331353733303333373139616431616663396131353766373035633562616562 -66636464656533626264313737663234633862346135376336613836343563653865613432643534 -33356538326464316133626236386436353761306132376664323331313230636538396531373338 -38626161356534323236313333333562653234313234383830373638646239353164356462626562 -30633863356266323831386530353737636338623365623665653035623866333338623065346666 -32616132613162316539386237386537613732653861376234346665623732326663653836643961 -38653465353739646664313837313137323736353263316332656436313333373639386239343730 -31643734666235613135396632353738636264303735393465393733653538343635313938636436 -36323730653766396365306163666135346339653132616566633165623865366234336461383233 -62636163613763353163643331396463396633653930623630663233386134653236386436353338 -35376463376366323032396365376262326466323766396338303633356639646263643538373662 -34393965343031326638656266363666323038346162623766313364333331376462366662633333 -33333265336630656532613135393863323264393230666639323064653462616637346434353962 -35646263373635646666363738613236396132633462653037633730386361303866633539376633 -66623762383562363936316663393735313837613138396531336332333535326139323432653132 -37316438313134623437343836303237633037396163636230643663343738333635386263323136 -31626365663261346132616136393734393037396339316165366634313432326530623337356236 -32346534343338326630393663623664306561353938626330653937666634623365633863306262 -65633139353433373137343033636137343935343135623231316264366436653935396334363130 -31333631666166623264363832306138623931616261643462633637636363393662616438666239 -36303738353734663231633730393232633934623161383364613437343664663462633339366563 -33633530633533383737336562343638346561376334656366333532616532306235666430666431 -38666137613834333236363035656131613763386534356665656662393065396330623634353035 -32336337303134663331646634363632326536636437636431383932616231643830396535353639 -37343537386638346232666363303134356365326366643737646661646235353434313135626539 -30333162303435646364646633353962313264656433363234356630646539666265643433393864 -64366236323831633633383263323263653034616131633538666538363461386662633333616432 -65646135373366373866653036366166616264633336613237373165383665643533653039373963 -61393162663339643466336539663839316234333133356533626538623336366233343533323438 -35663430343431313731306233376637643461383761373265396236396662393633363035623539 -31333738353838353531643963383961393562333736633764653634623466636438326235343735 -37323037383862666431336562653333613562653134663637316131366530363663363134383231 -32346231323962363534623735643830306432383935656661356562346330376132333034356663 -37336131656664626364333032656331326430353639623464383533383466353537363064323030 -32333261666633653933386139623936306364643132616139633332666131343635623933613830 -65633361303935326630363136393661393566396336626665313265636539393334323061313633 -62626536386661393061323932303535643632393933653931333761646233393233336530633964 -61396365636230333432346236653037343532336462663733636462346161346663383437306537 -35326362636362623230343165326338623865653738376338343432383538656465396537333935 -38316334653339316464353334326632346464306566383962333233376233636233376135656666 -35303838313763313764366639376461613932363634383532353936653632363762616163333630 -63656539323161333661393464326634336161316136653534316538366161616239326332366539 -65626236333832613034613938663864313261653631616161623038313634376435633232336335 -37323034636166343864313538633066376166363965393864323236343130636566623363393739 -30386631383333393431396237626132646434343632633734396564343231363634303435356461 -34316234333932653937343565363762363861356665666536663636666236643134643739306438 -33383165363666383963356538363937633136386561343938613636386136663563323538366439 -32646262383631326634383536333532346664373961323131356562646263346534643061623134 -38623034306165303661336339616436633362643164633563613063643561363135343332393666 -63636131343535663634326264643239363261653730393535303032383561633633393137316362 -37656134313966663062376665303331376537313132666131653130343833643634376161326533 -63326136643838653735303534326366326166323038306465383730356662363334633136613162 -38333964653635356164303039366366383965323065386236393335336431366439626565663831 -31316333323330353766386139393533666533366561653961626431376539626233663134666264 -64626536313834616136616130326561663563336438333262343539633835386263396537323961 -36363335623162333831626134663937366632316536323965336165616561373635373265313534 -31653962623439333563663661353636656164336161373365663635663766636536356633363331 -62323536656539343631386161346136333330613334623266643335306364363166323739386264 -62393132383661393938343335346433333833616131373531616638373161343234646463303561 -34623930323565613936636230343730633161323632646566333264386331373162633737656234 -66633366323164336430646135333939623065393031323164343131343563376531393262323939 -62613361373866336139373962353466653539623565356332653565623934633562636562623961 -30376134356533613037393863363265383739386165303034623237323362376436626434306633 -32303335306333346330313266303638666230346165633466383339356662663661646435363063 -64306266333763383337373439376336336136616330623736643034623165326165613866393037 -64306230333662643431643565646530323261353933363835383763316266613861386463383961 -63366366383464346235653563643737393732343533306533383464333766613537666430303439 -30383538326266643863356433646263316566666562633232316364616266343836643636303538 -32653239633436383430653661376336623039656332353636323439643333653066323237396565 -31396134306463663635363035636531303931373632363966656431656633653232343364323231 -37663332396636356433386534633432643266656366346365326534356566336139623831313134 -34323763366437633635313036633734356337383530326438356339316664646365613365353433 -65663162336534323039636235393531343938656565653361306662653137656138343065313364 -30373166636263633539363736333136393338616463396236643035363433373761653165306638 -39313137393935386337353864623738663635346531306133386362343263313266323637393466 -61333638383536333034316331313266656438373639303963313666333934323465386363623765 -36653131623530366562653466316235666331303631336665616532393366346134306561343661 -35643666363930633633396634306563353130323365356434346636633432336638393662623938 -38653562653365643730653931316132373637666161323534313965326136653936623739383266 -34613835336633666337613434333936663762373263323566666137613361316662663762366335 -66666465333062323139623138623262653765336333346261363731663736313762633331653030 -39653333656633616538393733646635643232373038346534323461353562613666313862633736 -32643865343462666331393531373566363635303966383832326534623865636637643937336465 -36313231313363323331336161363532373364616430356565613261363434373432643738636238 -32643130636663396630623565376438336363666362323931653066646231613635353233343033 -31623034353835343132623033333835303565616135333730663530643834303835323164303130 -33626536323763396233316632316664663065616563323765356130633435643563376331616363 -65336232343761363834643833633630633738376236346564343961333836346262663337646462 -39356263313335366239343739356537386535333136323566623437383939383637383931653237 -66313838386531663136396635373632656564633562343739393237666663626466323263396131 -65356138646566623233343366633331616638383234653633333361626132646661386635646435 -61633963643934653130303462383363383835626562363132333835646432646438336265333131 -35663163343062323532343366373637366539386264356363643264333233643864643964316364 -30386139666538333064663564313239366132666432626465333564633836653062396531636537 -39313336316238623166343432303735663334623136333561633063376265383031366462313237 -66623335336264626531613737353830623636333135663432393865366662363061356363363066 -31313963653663363661616235643235656234653166373564613936313634376566363831666131 -63633066363236313864363238376462353833306439646462613433643262386438633636643739 -32373063616466623235346432356362306163653231663532636230306630653338636661613539 -35613636326630333539633635633736626131646163326161353638303538653435393934356665 -37356438333031613536313632336134323761336261306431373030366562396435653739643536 -36613939366664623764333963633364313961646662656239336436343533326337623236303636 -39356533333565666266616661336638303133376664383939326363323435343432643837336234 -30306139353330363636353163376463666165373537316562656138306539333038643136646236 -34646630346533373262623966323263323838386234633133333461636431633465623061626562 -30643764646338333564313961333032366339396633386531366461653261303966313237343630 -30643263643763623539323439313866383831633036376562633830626233353664623630356237 -38363866313863373762393861306433373366656262636264343866636466336166373266363363 -31633564346639313137356239653036633334626235333734303562346435303063363137633261 -39393762333034623538653261663834323333346164616462373461393139346333316232383638 -31326162643263316534623765626230653430333461373237626137666566376361613466353432 -32363638323434303332656664336633396133366536313935326666323263653035356538303335 -33336238323532346634643736646335363134666438336165353537396134613331626635633563 -66396161616536323530323065333664303439383238643239386535353530643032373564326236 -66336666343961343639613033643531663837306631383433663233396366353732333566653237 -33646636323034303731316534326338363633383833363865653632663334653932373563313036 -61356630643162373432633335613139636339383231333866623538666534653265626139626662 -31393161633538346435376138613666616661383536323730353565313634376333353831653464 -36313833323334313037633138666435373732333432373566373135303035613436663664326239 -65373130666264393063616637633365613262303563653063616334343539343837363032386565 -62643166343763616630343163643061613137333530306435613562383161346535643739323363 -37643665303366653066373865333361623636633563636364396333363538393138623436613031 -32616462386332363563613663356361376231363765343031336331363564653438366435616239 -32386463353061353966623833643534633539343339663532306565653466353666613864656162 -31386637336463383332353637313034613631326365366637613037356466323961373536316164 -37663161306536633031393862376632326633366534313763383666356338363131353839353363 -39666561343437323535636133303130633831653133633836643035666564373965656561656462 -36313831663866623935633339366565653965376364346436653737336135383064393139356430 -39383634356333623765653432313634626566323034636632313836663565343231333533613330 -63626537636637663163326433383133373164333532626231346337396363306432316265353733 -33306264643766626431336430383361393932326361663336306436303735663063333565633231 -31393632363139346365346239333533346335353361313164623337326238646136326261323235 -33323665663630633933643239663862386336646633613438323962383936353036383138396131 -34356362633838663933626530356633623466333464643434653962356238306332323366316566 -62356138616232623830343564656535383330336433303630663465333337346661373132353733 -39333235343063306331656165646434663936353762303131656131366337656432616534343763 -35353732326562613038646234353366383230393239363633386231333236623633663931623130 -32306537356264313733633632336135333538353365373466343933623432666431356333613233 -31373239616136633066653633303439383836323533386437633738373232623933373633303633 -30646531396237333538656536646330353664393961653836316634373863623630396564363631 -30333761623032396164363963336465313038333331396465363337643933336137353932616636 -66313537376165306636386461343832316265393261316363383365326437393437316364623639 -31643563376462623533316634653761616635666633623432313664386131633039386437313635 -37346333656664306664366230303237373163623531373164646133353663393262333336316536 -31666538653861343165353931313531313138393737313637363037666633326231373538613331 -61633530623164613830623566303237656163373264313563363531303332303538386330613264 -38386165656233323830373531313238323537616536386166393736346236343533643561323834 -35303639333433326462646535396539393930356337303066363534393732366530313336633835 -63623439306636613437343030313136343262636431393265386631316466353833323537656437 -36323864623735386565653865383766633532373730303662633834303664643661633262343364 -32616264306539376662646563333839613835313937626663653038666561323166353232376338 -39353065393935653030313165373033363665643737656331396236363537653862383366363566 -35636133326135653761663930343937313961336332666432643539613666343461653032626334 -38313664653466656432636464313765336162623333323437623130393631366133393838663634 -30623066613230643437616363396339636164363230656136353737623938393136376237306638 -63373337623733373461343963653265313434363833616632653231383738666131383061663562 -65373265313461666630626462376533343931373037333161633938616566356431313763363331 -62366137616634383136343763373563666433663936373262646233633430643135616536393434 -33656431346332323762316435366634353831323633656133616333306534323532313462633032 -62343930633632393136653836346134323466353165366165356261306266366238663932623161 -64356532366262363865303239336530656535323864383537303235396330346165313361643139 -39643964356565366238656430323561373562363263646438633337343330616166356332356437 -31373237343739333037373135316239386530653836356433353761636538336232623134633235 -63323230323161303532613733653536663334353237336363386538323336636139643962306236 -32336366306464613761643836313433323363343464326539303739373634656431303833663738 -64646166343863333536646438653536653135623434643631656135373566306233663135613334 -34626630616231326535346335376330653233656536346263363662323939663737323239343432 -31306366353065363137666130353035356231663966393762346433396432383930366639393834 -33366362656461643235383232396233313862396239313633623066363130613866366537313136 -33376533326630616635666464303036626339386631333766636139366161663532303739636132 -66303436616537646238626637373865356339633264303035376136396537386335373539333834 -32373835313636326233616138623036366536383461316538383235396637303131666463666337 -36613337363662363462636264353030303837383163616232626563663332323163383932386437 -34656337356538356237373136366365333331633965333162356339653662363536346631653166 -32386166386662343638333234656436353964393165336632356339643931393331356166353762 -65343939613335333065323363613666653961323666346433646461663138356162663031396333 -35646239323962313764306162623737353831373738386463643261343532356634653935316266 -65363165633730306135303162333131633132343938613234306366313361333137616638396261 -37313339653834313637633932353266666433646261623033386463643663376465623563643363 -61373765666430356636613462363663383633353036383631663663393431643665393739343039 -35666262366636653563663532386232646566633132646162393463313631303036313465653135 -63636134376164613563666264666161643764383931303434353039316565356433663765306338 -38373736323564633931353136643832363561626163666534333965373237383637306438356437 -30663439626533643064383530333135626565643735356334653231643361373664663639333736 -61633836313038386530393537616131656536336232343765363461353734363931623331313362 -64636439353964323735633438346665303238303864656232306133636462303533353330393436 -32306163343731363730313133356537333665333136353161623036333565356433383166316134 -63343362626436393538323539366139346165663432373766353835643230303631323038356437 -34666336323161323861623962366137396233333133343164653238303032616366316338393666 -62343634613863653432306131353064646337386534643466386431663366656632313765353733 -32313230363238353131363437303034376566336331303565386166626566613934383136393737 -36386665343966326264326630613062666431633362393262646263326465323230623535333033 -62623434393131393337346562626639656536656266636634356264633036636637306535323737 -35623636626366343338373061303466366264663632376634363135313366663263633136326539 -36623738323038383662303832363338373839343063343339373363623634643365623330353231 -64363338353534396335333263626532633938636361353938623934623165393965363232386561 -36343333303037326365666366626466643639393463383462643038303638623165326334343631 -39393766643862353634636630376338383037326234666438393639636439653561386130613936 -34313965646264633266636539356333626435303138613835373339343438363130666636346232 -30613065356262653261386232663964636466623135333233656131636465623631346433303934 -32303137656630396338666562626135306562313335376335666239613836356566326262336465 -61636233343966383537636231626331313731363935363862366662326139643533633861616239 -33633663306665643763653338376264626530633639636532376536653331316137363462386335 -39353030653238616564666433636264653635646263393235313564316464636237393163656133 -31316438373263373439323264373561343433393866323535363337646338323161383134336537 -34646239633166343031643261353136616632646339636161386461613932613030323036366637 -38623735663931313337613536343130323331653335663261383432666536623466353133356631 -36326131323835343739303465373239303230613961393465653730343834393266356362323030 -38376161326430356132393438653361383266396434363833376365363434633835353530373265 -33613633656566346432623830306464613338383730616133306330373735363662383734383839 -31616633353730653261663738333838623137366339376430333631393336393361386532303465 -36356131346133336263326363323162373133653432336434333535633534373534343961383564 -38646432376364626163363935623031373962303737343333383063333762373363363931316533 -66313964656335363437306533386663323237623532663034646164313638613564356337333963 -34323766376633353962363230613233333934373236626633643337366666333562643536346364 -62343433376332656439373132643532363064623564313831316435386638623737373066383363 -37383635393735353461326238383562663135323639323761336437303262613336323530323239 -64623765653866616231643034373165636432333762383037346437616133376532396137316461 -37656330396134336162613037643536623435376535313537393130633365353934316365653663 -34326565316538643335393431393462623239613761316563353961356663313961366535336638 -30383465346462613165616338623266313666626531323665373237356635313561303437663733 -64376339376631396663353837353136383661336534303236336536373735616233646635376137 -30376137343731616230393937336433666434626361626331643137646634393634353533646336 -38636661646366316166623236363634313832373939643265323331313338306264376236633865 -35313439373664653733663361336265613930383533653936613665656132643363336263636130 -33656436383634333265663261393936333539353432393063623465663737383766663136303264 -65336365303539383035323761373734303465633432613064383439323137376362383233383962 -39663063626635363431353761336231356537616264643966623532323137336136356530656363 -37323834656635313330613534393961313230653363666539353936633239313039353664313736 -64643130336666396465323932633864386238303535643133356630353362383465313530633631 -32363936313431636666633833363038643765383332363035326332393734323464666335343133 -63656538363435376638333633663430633262363763356263326466656136646364323164643739 -62646535653633363739613631363133353136613363633136386566313035653537663562386239 -65356465663936363331613265646439646632393037323635626632353563306634386165386632 -66313437663433333737636231323633373063333264656463336234356235336462333234373730 -64313166653437653936613035376139303666346637383539623437623362353566383932393064 -36343332376534646630333739633361333930353564343965363638373434386566356539363432 -30613937316235313361626233373232393664363165363666353066363433623334353664663434 -31613537346164373036323733303565343336373230363062323039643234396562366662396236 -37643063626532346630326263356435373338323733396137366532646432363534383137633362 -34383531353639326365393739636164393738336264633337373162323863633939646331616136 -38666363623330393538333438643333653134366539323239376239346664643364653235333364 -33373565373838343365656362353230323532633361303133306536623961353236326165376535 -36653237373033633239616362333164386665306566663033643537326566323437383236323938 -62393534356633303666646366613962613837313536353062366432323735363231363638646338 -61626139396630343739643131633961353634326632313636323238343037366261663439653437 -32633966343464323033333331646530396532376436626534393665346334643130306436343831 -39393763373736306630383931643765326435303437313066623530666466376339656137333234 -36636539613638393366343338616536356532376338383366633331383164326536636565623036 -63653334623462633035313839326339616532626137613038653536616661313266653431353962 -65393534333064333834323165383161323362386162373230323139363834613063653633346366 -33343831373837623439663432636335323839633138303565366335626364653432326338356335 -39623432616334313039646361386665653533633530353031306632636432313430666163333838 -65363637313532303138653733303366303938663262356239633738313038633561373331663836 -62666530323361636539306136623033316533393639306238303563333963366665313161396134 -66323631323937396230383065396366633638303564623163376138303635623638613639353339 -39613431613537366438306536633531353463363462346664613635656433643162306462383736 -31386666333531396231663565353066396630356465666666356662653166363338376134623062 -61363265303766313138623665353534623461333066396536346162613965666334383932303461 -32666564653633643335353237306235623764303830373737333039356239633339353734616535 -33393666313932636162396264363062343837393964316666626535343365356166636130313863 -34343836393639346537313930616466316434386466333866333964623039386636613035326436 -34306365303139306634376539373565343036333531623863626238623231343232333432363136 -33663933393839646562306361623230336530353132643739393030386231383438643636313835 -62366363613730303662656138323865323936383537303466626435386137303931663030393462 -35313838303565383234376565613565353764386535336335646666633837316363386462333764 -65333133333866613564366434636264643565656630316331333765393638386534333964633963 -37373366333334623331633337323937663834326237316430316339363233663436303636323165 -36373739663639353131666331623134343133323338613135306337383366383734636132633961 -62656534623032383164343061383831343433333963386262613133326236643537316466316462 -36616465316631366335316434343237633965376164333931356162363230653638363135353837 -37383630356437306430346237613762346439633037363439633939633466383334386261633335 -34636163333739656266316130386539393534646564383766626265663739343738656631303932 -34326366613836663437306630653133626636393661316666373462346234393164363464663465 -34353635636532353662313939326233343933663430363631343037373032613964383437353632 -64646263663633643462363133373732396463353664383563383430663262616262363635376666 -30623039303637313933306436643731666161643863306438666430626230613337613562333639 -37333331643065326464333635356561316531623938393061346333336662303433343234316339 -37343330323537336639306336373539633436353564373837336563363236393830336334303136 -30643331333739386164623936663365373132656164613338323736623165383030306335323930 -39616137653931636263346465373935336134303135343231613836613937316639386338616663 -61373664363032663933373736373561613033323638643032383564613839653164376562616665 -61346134386333633136393233636439643933376435636235373631616130643736306661666636 -39316133616235353461373131656466663531666165643833303135383830333261383536666330 -36393732393837363737356366346633373832353838376561643263333830663061633434613630 -64356138626533626561626561643563663765313164663232363565653735353361346266393530 -34613061633734373734643032643031363864376137623665343064346239303835656439656232 -33646136613264646635306338306564646461376138343631363961653463383661336166633766 -33323530333061613363333565313732613663303934623235313566313232393365376434323237 -30663265666438623561346131313961313566363861363435346431393461306637346663663433 -39623263643438373039356465636334303930383836346337623438663635653066666132313638 -65636339313764633263666562316166313161643937383938333362646532616264343761383834 -39363064373639393666613234663934623532663439373631663138393233336436653335393265 -64666561623839613766 +66343534373261623234393663653566623035623166346433363032393362613237396262396133 +6162323264653234346363626437326236323662393338640a393139653064383334306363316435 +37663137376238653130336566626565653536613834373332393061393964363062376432646436 +6461666135313564300a633866663361313163663634343638653838373638626536396362343937 +39633339326162353137323532616132656461663362303661373032636139323736396536613661 +65393231613731396162306536363163636638333162383330313539393161306664386133643965 +33663634333135346561343131323962343762363765646431613835353634656164326662343666 +32373561303433393332383365316232633832623331343630636537373234646633623163323262 +66626563313966643665356663666539363837383964646461613735653734613939383234613736 +61653662343130313864383331626161343438353166656166356566333265326638353566346461 +63333132396663636262306634623566306638373335653064323262343634356338623633626230 +66653165326164653936393034326333343734626636353830303332356435626534306536386630 +66383937376636323133343334633463353238646165643637663864623838383235666232366266 +33633134323031616233666238303030646464313536393762636230633862303431333737333332 +64343564376531386565633739383732333661376461353739653535336635323839616532363461 +33626432303636333562633966666432383364393231623764353866626439643836323532383330 +65613030313636623431313434313533376232393466643933363563306332653163316334663861 +30316232636566326637343436323066346663653638363861386636643631363163346337386532 +63663866363733333634303266616332366635646563313730653063363562663362343263383366 +31393239383635323135353962353735666664663930396563326233373131373363633136326133 +38356433343239323261356636313234393264346137663361363336366437346237353164653862 +63646465366335333261653835626532633536373437623538316637643639343731633331663265 +36363433363038343437653131653935366437656362646266353665316134313266366563353336 +63333234336163373330653366323138353363643366626664393538326661303365626635643936 +65643630363762636138333365636636313731353135343466613232333264666135353039656338 +39393331643537353432303132626464633931373131626332636438663236633133333039663161 +66643734353234333763363230646530323063663838626339366634363665643238643739333962 +66636537303836383732373063643035386636303462363838346363633032393532343164343536 +61323839633636356530376130653931326262303234663338623635366264653732633463333738 +35346534656663653436306535323131363065663565313035363232386435386261373539313534 +37356437376236346437633062616536396266313836323138333535653264303937366466353266 +33393263373366623362373064303639333966656139663862663336653862316135303634303431 +61336561653366613963646165346635383465373465366637386134376135346531666334383835 +63366365646562656230663430326638656136643964396631323633646335373862323030383332 +66383664653736396334393238356562316136393434666433376138363133353164366539636131 +33653831363030343630353736363364643231643733646366626165613865643934373333623736 +64663737616435366566343733383339363339366539633764356337346137306434363233353637 +30626336653937646333393635336233373034373964356463643033643334323464366165656635 +31663931643834363161326566386239363034663064316562323932306634353632623137303466 +35336461663032323837663861363533623061333334323639336532663861366161343031663933 +62343462366236366434656430663264336533616632303936656439643862396532303734306637 +31343364623638363536633233333432393134363239306137303065323632623230313963633934 +32383736373663653533343738393662326363663566373636303431366264666433356237373633 +37333635616331356233383464626631623130306161303034343966333161663663313565643364 +33303963303531396363333462316331366430653032356661633937343937323262366165353539 +34353264393334336161303237386431313534393633633739653838306665356535636637373939 +38663035346337616261306231386461653931326662643536316633333031343435353432316430 +38363731363062356636383133386134393430333061303231613536636436616361653232636666 +33356164313232366234663662346336663636613662373837616638666236353636636536616563 +66363464306364613362643564396562336431363832613338393439353933346262326533353665 +65346433323838613961383337633636613336393161646232646233323262643866353665613239 +66353661613539366664323733393834646435633766646361656632363132306665383239306536 +38343162313435363063323136346130326131346462363631366566663137336333303235376130 +33363263626261333437626333343830336437643064343538633839643638373566366661333565 +61383261613866333764653138646361386331316562316339313665616664383831373366616230 +61613465306661363561393766633833323466386564313466616564383064353563343232303735 +38386236386531393966373230313862303161666239336234663036626463663130393565393334 +34343966353231303738353332653765336265323766303534616161393139613866666565663137 +35313162643331656462643635343133626230353935346265373861333463626331336563643539 +31343836646535396632666362613364636435326561313833643236393232393266623635353931 +66373035616331633166666637653465396430623266343532643930643864633764336439666537 +35666564326363653864373533313533656232306333393836653937646237373531386438396339 +61326135613736363934333637373563383437613732383635393233306533373530333035343839 +37393632653633656665373066356230396132363465386337346335386366663338623562656332 +37376533316535623238303038646532313837326633393230633736306562353432666166373365 +36303836623231656132653230303132383331326265336265626138393664373939663332386539 +63396664616463613630303862306364653833303561353431653934323963616635613363366663 +34653864396261346564363563633538343738633164313062643536386361646637666330643065 +35376162393634643237323239326261393236353930316266616464323233363739626235313863 +38636262616236396233623166663463363863373933386261383338643166363533643962636430 +37363335363132343461653664386430646239393739393864666466666439323762653763303331 +30353766336666613134373364386438343264653237653163393833313735666663376564363163 +33623335316461366234623736386537326462393861383135636430393135373361666432626437 +63326330663734663462363466623865663031323165663163363931363639393231656661396638 +64363135376566393239656530623238343633373432343133343936333133383436303137623037 +33316363326630356334373439326662326236386261343834373361383134373864653530313437 +30393162366133633637666464303437386165356137343734336263656566363864333066343432 +65393363373135656438316665653439666536316234626434393466663461356461333866393064 +33313163663236376336633839326365663663393837383261633862366132623362306462623531 +34303266313962303464633138323262336530383863316338663133313938396331373361346338 +63663935363438633361383637373766646664393266323631376532663430623632353535383064 +38323863663639636361386637353466393364383532396430393635643631303462336164393966 +63323163386632333663363663396230393034333466353364633164356638613933646436666636 +37613939386635666631363138363732626636393030663165646538623264356263383563393535 +36636431376338303731643034633534366164663866323764306661643632353636663930316330 +30653264323661616431396164313731363631613862316561333336626266303030633938333764 +36303661663337326631626264336530336534323930383633316134366665383933323461333133 +36343233326431386537356436623364383863373630393961336538323766616237663466323934 +63316264663738316135333434333630646464323638353933353837333437373161653033626434 +65353066653731306462386632636432353064396465633632653937336631613962353364636430 +31613931333631623831613162353361363530383038633933303538633939653034313237613262 +30653563333039616465383964646662346338336133383262363265386437386232373831393161 +37613863306235363933653838643935663165633566663435303934646133656434313339646166 +62623534353532633239326534636534356231326661363330626666646637363535616332386135 +32373765323032363331636537323066363565666133616466323635393563373730303531613532 +33396465666664303264623361356163326363616662323930386131323763616239656231646465 +34616339386639343739666538316334386566363763376235333233663162613962303863316638 +34346535386430383264346633643764623665373135636364383631393662653562323365353161 +65636339653164393336343962633365383037393966393331393730393335373039653361643637 +34303539663137336230326233396263643563643166303530346361666566663036616137643039 +34373062343539343137356535653130336136623663346166616233626238616230373464613966 +65363739646138383361393335353337393936616562636431396538303636653334656431306565 +37623764666231316164346231306133353334396161353638373436663232623561353536643636 +65656637326633393064396336666532353964616435336437376332383334306461376638613061 +39343934326432356633343162353934363066346239356532353035306434333535336534383130 +32666631363665613230376632396162653933613432613837353431643632643530333836616637 +63353661613663363437393530623136376564643732386262326435333636643331313536323962 +35643765303933396166333365643262393562303862656538643836336630363132356261646162 +62666339376635303538643464396364336336663333616566343062663739646464373331333537 +64303862656138326133343165316533376262376137353963353530633037313065666666656566 +32663532646532663337653162356135643035376534663134653539326634643738653535373633 +33363661386237623133366336313331393663343736373932333463303937326165356330343038 +39373830363737643737376663333837646336626165636361326231343961316666346331653263 +32343133306331303266353835323065313438353765346635633232343236646234616463303032 +35356534376564663761353261643530616639313162353131303665326137383238396163626461 +32316263376133313238636438396639303064383265653135313539643463616131653736316537 +30363964646436363838646666626662626233393763376230313434653836383463616133343136 +35313536336162386330653563643439313836666463656233393439346337613536303633303664 +38303931333963323862396435306136306639383464363131663935393936646234366362336461 +65656232366165376236623631363862386464613337313165643461663333326631373335623334 +39386466336363623463626634666136643563653930633864363665313234363534646632346637 +63303766383065323739663163643837366465396237393532333862393036323863356262393631 +35336165626134623734333631363466383333376239643637333061316433663538636666386465 +30613663343862336462363436396237303339333261313866346432353764666366376633366339 +33303139303661643836666132386135366236323061393963366362336130383635313465396135 +31653331323131393463396165666163646536626333383466386537663863623531316137326335 +30323930363661343832303262356262613934353163323935393362303837616330653231333639 +65356334363534396236653734326134666333633963346232663664613631333730313232313339 +31616662383533663330626536363030613139656431643561353137663465386664333734386163 +38653636343736666664336361633834613965616264326661633862306330336538636639353864 +32313932353065623062326266323332376232356436393439316530653733326536356633383162 +65303165356162303130316132663165303265623439313465386264393232623333386130316537 +66376430623664656163643036636662336631393361323438626462356133313038316439643536 +39663965323465353435303739376130346666623736633239633661666234373537336534346339 +36313638646563616565336633303934336663323331356135316232376165353433326461336461 +39323930646530626332653365663131393761353635633133633534353061326264633233616438 +61643337623836343036396631636366343333383461343863613763343331373363303833393539 +35663131653964343666313930633935343932376562343263376439343866366266303162363638 +66353037633766393565623561613934656435356162366461623630383334366535346632636337 +39306134303832363336373237646465383837333766356233613062386130386236613331643233 +31303432386435376239636237303338636538303365323130333130313165643861623636623262 +30653561333761653533313230323165643638303463346336323131396539333935616161636633 +62343765306163663439616465333638653730313665613337393431396532333435383135653033 +32373139636265353065353064346634613638383432663466636231623434386534313631393534 +64343261306363656461383133383866313764623864306263383666656563313366376134336666 +34353463323236666334313330636330363536356235376364653738366239336434356165343933 +66663038663761306531613831633731343632643666323333313266333065313833343666643231 +34636536303635616231616239616235653361303030643036373831333630336432323238343966 +64646164353732396233636136313434313035323735303366306566626634306137653830313762 +33336433383530343865313736666363633639366661383664663536323730643966396666633434 +64633831303532343031373436383037353531613131386562316663316161346433326166353336 +37366161353839656137613865363531396137646632366561373533363131643032653861653533 +37613737663639646663626232396536626631343733653764373139346531383738636261653963 +39613631613561353831393130613364383261393765353537653764613339333837303163656162 +61303561323238326439356266666366653165616532366231363239353532643531373063626632 +36363937653465383538616461623961343434656331616431333462353264363632613865333863 +38663161316631623235353136343564643463373361303861356135633833363162306365643963 +38656564383033333936373732323238303265333736333463326436663866343037656463646637 +38383866386462663136366435386636303262393934313733643861396435613639323237613438 +32386566656433346164353730633731343065636539343764643865313363386431323766383738 +38633332356535633630303938663233623231303835316161646632303264376335363438343433 +39616234646465336236333037666262623536303966613766333231386636346166663033643633 +38636134636333623934653239653035626538623262663039313838306165663331616662313661 +35383936396339326265343665386135653262323132633036373431323561303037663561326566 +39396435613430633939663362626466386138653636616131386135336266633636313139623862 +36666235323538333963373833323365373437646361616437643934653462393134376365383737 +62633666363863323834376537316334313136616365656434616235346265323461633161656630 +36366331346666346161303631396637366463636537333736383535346161353061396137373662 +61346437343063376362303337353133373833303231633866656238306337663538353066393730 +66653238363234336634373736336666366337666633356335643237623762353239363737323466 +61653231646561623431333338313562303665653734333861663937623232353633346461373639 +64356634653138383434613161623438386664626433653131616637323133353366323738316634 +32366138633331346631343234363638626638313839333937636461353562623465353030623233 +31353164316430313933646133643238663530666634316433313965373439616539666435383937 +34613431303036653134313839376237343734613234343036626430386330366331313062643936 +35626635663331376637386430666234333135663232326465313864363763643766326432373661 +32303830656439616262356436336436666331633763666637643731383061616638323562313962 +66343438616133616639323439623563613561646436656530313431306131663861393035373565 +30333062356564653164333235663736623063333539393734373463306631616538396562383933 +31323736313231356135656637366630393566313037376565666532353634643833323230646138 +36366466666161386637333765306465386263333966313033383832363931303234383666393039 +64313532643732383434636161643433343466353930656232356362333262303338643034336366 +63376439636665396165343162633339366566303762386566376461393863623932383161616231 +66663539616230373331666336386235663837616631393639323734663431363735366233303634 +65356432616665356563306433393561653632616134663165326530656662316665636165316134 +31396637633237346165396434333135353163326262646233323531356464393234383061343537 +32613736356530363137646439356433633630646464653333336632336332616235646439643531 +62363234623438343131316332643039363065343238623538303431336632356534353336376538 +37663538323865373537663938643965333830643663636636383531303966663031383832316635 +36316364613231626263396664346233663962386463313663383834636634386437386633303034 +32333636666430623033303135376263346136653837356235316364643934383332363062393433 +65303661353733373434386632383339346637343638396432396439323066323931323730366538 +61656165343434333066623365336562646361646665386431323339373662333365353233323137 +38613732303739633936626531373039323230343133396335613333623764336530626632626433 +33613237306465316237663730653061373635383037343932633032616661363233326131376635 +38666630356530376434323934373364656639353061613437633564643864666435303437396361 +37323866646466383963363638646564616633313665383234366135353763663939356132623738 +64376134316136333133393733356534356263633935313738306436306363633461633638366563 +36333038333732353466633061363132636266346637343935643437343935306432343236323464 +33333038656439336531636466376362616161646438386266356263663763663235393932353035 +61393437333961663736356637656536356165643362663332396136633136336665336461613062 +33306336613765333631353731326564333366366136663232646132363865386166396430626366 +62656630393064396363626135613733313736623762396536623563656234323661653331633162 +39323665336462303364663832323437623835613633643263616564613832336337343965613838 +64383761326537313035326265323963383134656162383766643766633361343764633437316232 +62636164303632666339643030663663373734326634353333373933646432346237336531356637 +34386331383037376131343835636536613435333235333062326430323936356639366534626236 +33356234666161346131363231303762356236326665623962623664363538376537653131623339 +37656264343961393761363463666430373234373831626362633233373966633330396461313730 +37343263376362666236396266663533386665373931326539343035656565656434666465343565 +36326231366561623133346264306335653831333333303464333439363932656437613439343265 +65623164623234316533623639333733616638356562366265636363363263323433323765393934 +64646664653435343033333839636131396437383934303539363035646362643966373637363831 +38666332666566376262306337326562343732373937373435313964623366383730663430363965 +35616535343364386433353734323333363633316262633530656136616138396632393634646433 +65303633653632353161303430396430666665633766653333353330313331373730666639386463 +35633962633834363264623930303133323230383762323831623335333839366161653762343434 +62363835393663616539313262306331366666376232313733623931346431306536663037386362 +34363031613264663061356431663264663932306363656462306338366163643734636537643265 +36393731343364363338653432386262616266363037366330633338613733343461396462656132 +64623831653034376238666430336336383665373039653037663262663763353162333266656137 +35393535313133383363653435376663653963613261323364643461353962643038636534623962 +64323261373436646438326531393431343637333433313835386132346333393539353938646235 +35373864323163613631636164623665643032663734663439336264633861343838353838643465 +63646262626332646638656665643639323764386236666630353738356633376237323734613437 +33306130383032343835333961396464383037386435386530666362613232626439303661646630 +30386164333138373933653837356265366230356230353432643639366364303338616163613534 +30646236363537666236313462373130363765313936656434313830663638343431353739313232 +66646666653561623030663164393133353864383530336436373338626331323330303465643930 +61613837303131366534396339643435623130656330373361376438316430313733333038646637 +30356666323439353337383964373561623635336364663366303561313237646231613636313864 +38623830323239336639383336363333666131316635363866373865373834353031376266393730 +33616162376337643162336266376539363764396533396336353639616537613361636536333038 +65636263613535386463623631333864343161656564386265333137633230303563323965343338 +39313230383533343431373966306434376465326434623630303766383963666236626430663564 +37383930363765313533383261376130623364303563363837323637643630353230333032666532 +62363339663038383137393264626432356331643766313837636533303361336437613536663737 +66333432636364396235656361313634633261363338633463626630363765366366336339313064 +39363638393063653831393537366432643434343338643539333432373030616236636266393432 +33373334343864643932386338366136373632313035613865656162343235613135383164363634 +30353566366164383037336462663835373830376463636564343534356530356631386533653964 +62613439393435336436333666663936376161386231373536393030373736363839313661383263 +39336236636532386634323838613964623334383663623137396635663164376166333935353038 +37393266656563623230643362373534316439353966363233643339643635626633663539373564 +61306237343030333735623030633539376130626166326539393362623965613835363032663132 +63313263303330373433623565306635376264623135636533303662343562623765373435653835 +30393162343730613232623233366263656362336265666539633262363835323230343065623738 +30666563336633353731353136396139663965616365626438326337393336363535633738303661 +64383737643832336165656433663131333238356465623536313039653265323036366665653239 +61303131666432323862353339656565313231633737363938643934643832343239666537386135 +66393339373232666466383636636464646333633133633762376531636630646432396135656661 +65393630386239343666386162393233353131353561656532613837303261313264316436626635 +35666339373833313065346334623135366265376466303832303935366436333964386131643964 +32386637656134613838663633373930346430303066323966333338633864373063626630616238 +33613461356434656330306339393434653732366631313461373937313561343038306566373634 +65373339623663653562336437333065613831356665356231303765333435626466616631363034 +38316435323836356437653536363062663262356237353333353962666163666432623164653936 +32393465383839366130386663393263376365626337383833666535326333343264376530326562 +34663832633832626630646430393665313835303438316663623833336338383865613238616662 +61353033346232316465613166306363643634643436656138326536363737616165323734383666 +64653931336535643363386661643135623664363666353439343763663539353065303263326337 +37653437653635663465616436343964666434643831383737383263356436613262626366613038 +65373663316438363763346366346439393734366339656534383937636463613737653331326161 +61613965646635383234313864653636313235313039376337303434653935333931633463333664 +62316632343931353265613964643966373638383161366138396234383866376161343765363532 +36356535326662323863366661396232353337323264396664376463633235346665326133326365 +61643666366661313461346465633737386637626362663638333535616261303238636565306661 +64303136663730313631306366306131663261666130356535333865353230366338646238303437 +31356230616463643562366365393739313937363832663930306339653761303138386335336566 +33383330653164613332333030633331393935626636313239356264326634636535306331333962 +31353235356135306132616635623264393837396662366663643334666436346236343138656561 +30393231393132326366393766663139366333346632333537656466343063366132316362316532 +63333032316166613065393931636262383431393265373662383739643664333139666432363664 +37323761633538653265353935653761393765623932303032363932313862663064383065353664 +63663664633438363031316633373635323736343565393539303035643262356365306231653730 +35313061326531646538333134353632346439373838393638646337613363626139663234613036 +66323935626164383164323266616333653735363235353364636130303261306635303061363461 +34343864636235356434613232376261633230373636356465616566366230383461323131613239 +32306632653834333766653231333361323864323836663562333036313836313362356533656264 +32383230323438363034333831373236306631643936393432623334303139366638666661616661 +33313266323131303930656530306261313631646434303333326530633138383163353661636239 +38326561323838393466383036303231336366363163643733366531366465303032313163386262 +65663431383061306134653436353638633639393466303562636162373764356365646366333338 +38306138383962396235666232353438623435383334323632396432383539373266623835386134 +63613164363861383662346638326262633961353130376530303332636231343461373536306139 +34326634343038356361623737373332363861656635396366333630636563663234306236343832 +34353733383930303262623136343732376139336633653761326634333739326564386232656239 +66636134323035633133343163666663636439396335663132663261366465663836626438306432 +66396138363530306534623664333934336130636564336665373864366237373530336535663536 +33636335396463303437656130616565666338663737636230313935373335386332366333383139 +62656364376564396466643236656337393034376562656635663134646265383238663034626136 +61386635343535613865663464376132376230613434616664343635313130363965366232363436 +37316437333736306364633636303435656138313662313637356561356634306366623064356163 +33363835326461666130383336626662336434306635663363383063306331316439363361623364 +33373937363937616433333665363033616437343265396235653232653863313630643132383137 +36623837646662633161623533663866376430356634313464383361653731613732363436386666 +32376663383139393133313139643930323462316562363535613738653539336261326162626331 +65306362636539343063306336616631396536616662623236616633623732306161346235653861 +30316530316266376465373164613832623332636133353130373236386330336334383765373264 +66376134343364383330636538666461343336623537373534663531333361613366373435653361 +32626132663837343832393439666431623133303930366238323739633665313364306436373865 +31386639656261663361323564343636656162306133393865636136653130343161626630656662 +63376132346562326431323334343136343334303962613532343131646633376163643038353431 +66376231393235363666363531343365326261633031303132356632343261656162383763316230 +64623033336461343866623739343462313433336263343436633033366134646632303662393731 +66336431363164636265613761666532333065356361386430363130386231653164343664353632 +31623562393161633335356261656632396630353036663465363637343264366235313739633631 +64303662643639386132613134656432366662333037666133353435343137343438613238663666 +35656562633065613534626364343836656666636430663934663333323734633734663566386266 +36313163396235383436373138643236303531353938353132643562356537373439336238616530 +34636434663335356434396134623733363732353866323166363536376563366233663235376435 +32333261383937303734393363666365636638613362666330366562306564653330356137633738 +66313266316433626239303362646265333664383930613539303430613131653839393162626532 +62303461613538343137336434633832643735643430623539313662643661313134653166373335 +33303933653739316433626363336638636231616361336438396338346333396436656663616230 +34626330396638326333303864626433396630626364396335393230333564396431393833663433 +34633436666266353037376465343763356531323838326264636530323961316164633566623931 +31363939323231333738333339373931323663353832356337616234393935633362653536636638 +31343262636533343566306633396537333331653438653032663533356631313339653139366333 +30383138343263323633376139353332646236356566363865343532376666646433613761613039 +31643965346335613961623530373466353032376366333939306562303666653830316538353964 +38353766666230623332353939663933343035653866323633373735636436636463313337353266 +66663535343435646238393937626634366531373732323137376536663138303839326130633764 +62343535383531346365306163323638326465643365353964393638633336393536313362353830 +33303436393237623530363235343763626538626238346630383732636163613966303236626137 +66623262636365316664353830653539306532316234306566646462313536656238346661646566 +30323833626365363431613962373939316463336334323761356132616534653862386262306631 +61656633353239393066373437656631663765666563366636396262356237393139663763316431 +66616466363330656231333466363337636534313964356561393065663038313763313962336438 +34373831393734353134386131326338313664396535613064626137363433396463376336653530 +37393537353661643962663539373938346134393461616262626264353564646330663334663064 +39326634366163653866646235343435646636333061323636313730333863613465306432656665 +61636465353736363538373033323439663838633934613734636434353433323738643165656263 +31336134386262303965643365343137363436353634643637333433663763343230323335643633 +30306665333431616237333536366635356363343633373061643266303437336234343333633766 +39313839306662646262663237386138316164643531623634666364613338623230393061323764 +64363736383831303931336435623261373265303166663330343262383334326332363034303339 +62623538373132323638366462376466323336376465653130366139346161386338326464343234 +33303839366365373538346534333634393465313064373461616566613566323764353134373234 +66623231303362343161346166346665346463313032653036383664356135643361386535343462 +34356236666663393235393465346238636139653261373032366236666436373261366135376131 +33613530613739386663636463393135303339393935346265346333633662616532643039663066 +39323331353664353965623763363664613739333033633835323534303234343664306633383131 +35633664353165616239663930653333613431373132326365623235326132313064653031616136 +33373465323133383736313063383637316566333738386431386631623434613339356138353431 +62356561393133643964303234386632613737363161643938316535383934323936386666623838 +32353531343965323035353666383136633639363530623563353164333238313134663631376437 +33303833353933353237393031363939346533343130616163373261353234663966623533306632 +65313137316431333964376138613032623262343736626433336164353237376465616662313535 +65303539393135373163643230366536386338306163343264336132623637306334643465623630 +31323065663065633533326235636434353738353830393637333230663239656532306666626464 +30383164656332393965326234643031373937333037383330353238306465653362366237663663 +35313835613531663862373534663566643532363163326231623432373236626639386565643833 +34353264326664346466313134366430303265323530633938353961666338336566636561643665 +31646164376133366437383030363436303262613666643362373434363965623833653464386637 +39373462613035663763633232376262303166353936323032343233303437393637663939326631 +65653533333963303662353331306162663237393430373233643234383932393734666332646535 +65346261626130653037303361323833303235653363663366373935656366343936393232643530 +34636564376365653263323831323966346631636236663532663637386562616663646332353163 +63353537643465636438313432303566663865356534326564333631666662663139653937373365 +32383636396466633138323333633965386362326237306162613738306162343265373535613565 +62613030383539626464316532373131663833386534373063633665306436353565303432663763 +37306132323462386434346564313462623235386137376664313831326339623264333139303139 +63356536643031373963626661666231616262633936333865353939326630653366333431663038 +65653731383963663731633332383465633232386530626566336539653834383232323538306438 +30366233353162353339623561363336333038383532376533333337353762343166396462643465 +33623833633461346466343762643864646632666533623461633935643937343139303735646530 +35306535356331336366303564633263326538643564623733393161633538303562653263386634 +30353931663636333036653430646263386263333839613463383631636131393238353339633435 +32393564313532316238613962326336306537356662646237313932393038313739366433643165 +64356637626537313131336333623265373438653066393665323662653561646136663265313166 +32616538623866653666626337623030313635323133373537616131343763633434376437646365 +62343939613732343263333636393139303962336433663735363366393837323834346232653439 +31343331653230363933656134646433396639383361646139343430613433623636313132313563 +35616336386438353366333431636432356537663033666331356562353736383064356530376464 +65306535643261363533373063363335356333623866366532643038366635313261633938343764 +37633034363131323336363133333264653133373332643430323263336336336131613731343034 +33343436633264333562343830653231376239663234313134306139636664626363373334323136 +32336237346136316431313061333739623361363763353139643134326132646263383264653930 +65656230613563646466626435316231316330356166386237663330343936623466386130336335 +31616536336166636639323638323537316263653333353166376331643235343064623433363434 +64316566393865633231323766333662383034303738333061396665613361393966633563633965 +32636134663532353634663363323963643333626530306461353636316264393936653130396139 +66643939383262646462376462323737383739643935383637653931383561393838363363626430 +63376239633166656639633266323661396565623135336134663930396232333166303036396238 +30386434323336326633373931376339346235656139393230653338666132613064363664363232 +66643735383030613664383361613030633935326331386435616264306338376430316533636461 +65333363336633346366623162633835313063363734353030373133386462663937316563636463 +66346539633338373630326461643032346264353435363635653932343161353238356137656436 +30376135613733376261663836663839323565356163643935336433666165633935373966636663 +33393965663066633964353166386138663830303531663939333064316639373464653763356134 +38356233303833633039383438326631323962373138326134633061393866656638623363346230 +61616130666534656530373566663130306431623565613761373033313833363234373930303334 +65653436353564346233363335383237623535623832326534353132383961626136623937306136 +63366338343364613131653635393961336339663062653438336639323833336263616130356131 +63646239616662373237623533613632633364643131303238383366333563353734343461393666 +64373037626464393962636330366465343561326265383762363237356563626139323836373632 +33643236633064633630373731326130663139393835303133643938313937373535386330326530 +38346361376563343438623861333338623863343538336338656438303738353362613933363533 +38336439643062373661333862386561323432366664653864376634343962323735343036323834 +37346236346437353031616333336533623336653566306466613636343261303161363663363264 +31383532363739656131643765323865356134333738623536356531633565323134386332616461 +38376535396532613537326232333030376430666561336235643039343833353363616638613638 +35353537376363316231363439626435353531623034653864323531323465323631643333633764 +33323861316161636534363035663763376237663739666439613134323762653166366164356635 +35363939303637396366393237393637303261343964306135393837383737653031636535313936 +66663438373066383430363134366162636132626631356364626363363931353762666135303230 +36346638356133653765373363643336383838376164663463353338353066633336373533393736 +36653033616266336366636666386434353961356231343966626164343466323036643334396364 +61636333396165623839386161613430386536363238633462333636326531343234643663643134 +32306635356562663937313733613438653765356532616166376365633639316563646331333231 +38316435313063346639653834626230383461396264633838363562346562333139313137326233 +33373563316636633666306261356138373031383565316664663636653433346561353535343463 +36313564353838343563633230376233656630383233623263656332303936323936383638633533 +66396561346563623064356161643264303066316336323663613031316237353735326464383938 +31336236356637313537623934396165396464363634633962636236366338303835306538373262 +35323462323438633532346135393466386237626137393561343732346431613330666436643139 +37343337643465636637333836316331306133383465663264333436626362616262613237333037 +63613531616338333232383031333162343161613937633238643239323236336331643630326539 +64633463666261323435663236366133333962323431656636396565333332396664363661613132 +30646339626535623532303339363762356462323739666537626239333164656530653861346564 +34383831376466333665356366623833386632626535626235316261353664346261316538656363 +37393665346631373839636238316239353962346138623464633832613861653861323065383062 +63613138353031326236346166656435333736636364363233306536366161383863303166323963 +32653439646565336431623230393765663530353863303036343863363130613330366535366432 +30666236326230303333366139343232626430343138343836373735646239616139666161353930 +36336535643035663863383561323061386639386564653835353438373163396632313561633232 +33306235643435633366373434363461326439663735623662363566316436346238316338663236 +30356435346331663137656364346139333162353935353631626133666232653639643963386235 +35343033613935393738383261663834323234636631336266323039376266666533656365393565 +36316532666331373861373932373639623461373465303161393039623235373836613061653466 +36323732323866653731336538646635323865306333376638363762333237623834366632326238 +37646530336461383732393435313537623566363666316335643434396637323736633538373930 +37363564333339623062386565353164353166633235333531666437626562313837396437333334 +65663838633064333639353766326333653736613935636139323139656266346563663836383631 +31613034643161633637366132666237626561653931353839313036623466623465636631376165 +61386336386663373361663939393930633338663335613864646164373063383264333961643936 +35616539303062383337343736643130396633653464313535626638316430633539353733303862 +36643131663935323132363335333638393237393864623033353937643864343936653935623163 +35653061353639303730323639623436363737363161626566626662306263623361326363346239 +35373733346631313036366333633730663965633135393464323636623630313335333561373665 +38323963616437376132323563353438336135663035663765663336303863313537623663303030 +31666432313665356262623963363333623835623062356163666437636430333461613266343566 +33613537363334656666663165333834363462373064373964373162316264663666363332643662 +36636366323038636338363235376132366639633938663362376139666165636237373330373832 +31316561623131383635326666333635663738623064643534633362656439366662316266623137 +66313833396439373131666434623233383830393264316131366436323230636331393734323761 +30306364656162373739623235613834623366316237653264613134333931346463303533346238 +63326535346366303662626636383765666438336665303636646136646366396633616230623935 +64663866326537636630316637363762656534353934316632386438353361636438303530666338 +63356533363838376339326139663634653066313835363038353239326238633761363731666133 +66623566363564633336366663303965393336643333313437326563613862396139643862363432 +63613662643135396539653832383036386330623232336231663838393966396337633166353135 +37636463393538316537356139326138333437396135656333643432373064653235386432346335 +30306261646530666238343130626662643565313732303630663164653162623939323066383665 +62373231333732393639643136663864386336303232333961313036616236643635623830643433 +33623537633438333931643739643563643332313266303436353338646136373637643930643533 +39323734626266303837643364323638343562313139663036343837653834626665373964623235 +62356437303362373232626365306561393062633266666533613066333663646534333233393331 +31333666633236666335306563333965326463623533396162336164613465393161316537623434 +63366536626130616363643066313233653932623363643066376466393937383063643861613532 +32313166383430613262356433636663323732306131363566363136623439626434393736366530 +64653561333364383832393636363830633966333961333531333732373333323133343835316634 +36616437633831363437616465336362646337643266663861376236393761343563666164363030 +33666231336632623834316430383336316337383163666631663162636331323966373261343461 +38613562663166623032633637343962343061373939306236363238363432636135653036316539 +30323632336363376464303038303162356562346561343830373364616332633438393732323838 +63633237643531656233343030653433306563373135306337643464353533663566626236356661 +37393334333133333534333132383935363464343037653664376538613961383766656630616263 +62623738306433323237333537356336353634633138383038366239306562623434613234333531 +38626131666163323862313063353764326663376636396236366437326466636434346235363466 +39346261353538343139643133666462373238396636613731393635303938346235386165343931 +36353731636661666665346462353036343831656230323234663339393235633564616134633233 +34396364306332313731643439666266623831623135386437353731313461633739313535633461 +66663837303064636433646134336539326533633937646632623931356462396364376166316164 +38633134613038366439346164396365306665303238363430373237636161313236343836643861 +32343130623133386466663535613132336662666333346438313236633861313464376135663938 +64666335656636373165303565356336363334333337653461323766633837656363633664346337 +31333633623335666566323364633233653566613730373737613832353066656135643831393866 +62333664346566616234353663623935313165383466386562386262666432373938366263346365 +33366632306362333036356335666333356138663836393933643438336138353561626461353238 +35353735303030633466363765376130656635326435393361313865656235646661396135626266 +37306433396539306163363833373962646335343735373335383738666133663163346333333063 +62373039313436303335626266363863343239323666313164623738636131663161333931643364 +37326265303730323766343232643162613439616433313061383365393561376334636535313333 +64303830363863323638373236333939613337643335323131316132323039323936323761373763 +66373736633435383465653834306338326532613430363364396461613638343538666364336534 +38393235333262313735306635353132383039353436303463376362313632356664623165316437 +34653838373830383263323362383665373130346261366362343662373730663763353865376535 +37653935336632666633376662386564333937383362363365383434636130346362366534356563 +63653633613837303064653733376261623431666138353735396538643165386261346335663061 +39363962373036663162313530393463336230326665643833366438303764303364366637323230 +65663539396262313337346264633561333663363366303061303265343463396135353233646536 +63623638376130346131316163616264396439343437393732383731383433366638623736653564 +32653233383932356165643465646161396461313261393563303164663162396434396238393465 +66626530626262333137653062323235343332303132626237333065396139633938363130616663 +35353864306433616562306664323530303062646135356130343634326630313535383065653465 +30373530346136333962313435336436323135633238303032303737643335313339363134376138 +38383034646237363138333135306266353238656337636131646664336463633564373638343530 +39626434396534313664653162326164393633666231376263346234393130646536663238646265 +34653634663832646439313038326363376234333864613062353364346130663533346131653162 +35323331356565373164393831356237383339376633353566333138313065653833353437656238 +61343364323363666136306664353533643665313031663739323864306165366430363139303134 +30633431666131306465336138636665333134636461623364663566316461356232633635653532 +65386435646230366234656166626163383661393363646234633361343861376231323333613331 +35386465383736616366303631633066653937393363653533666662393739346331333737373533 +33376331613930336236313331636637313364353265366265353836336136653865653430636534 +63623431623663663964643930323630313730366339373365346331356263393561653435643937 +64303663643234313464653962633033303236313830383338353634323437303536363664633061 +31646265373366323962353365333166323731646634386336636563383563366438656237323539 +33323030313331653534663232666461343534626433353431383439626533343437643962616563 +30643662313165636662363461336430663033336662636132663131653930373762643433376662 +31646666643864376335333262353861633565363231663634653438326264323661306234363530 +32333834653738333035386266383036323931616262653833656235656335326236386531376435 +38326530663934623831343133616461306163396237393963313263306637356136656532373739 +65323562656336356264663436626332626164333935646131336430646230353133633834653833 +65373064663434626431363230313661383631363534663666306330373766306261646534323765 +62643062353832646638373334376266363739356265373462343734343035316562353839613165 +39323035663637613061653561663531616430373836646637313864373361613261656265656164 +61353862643934383064613139343737353137353762613765636236366630343265373334316534 +62323235633937613763336330393838343935393531616234353339323335353537623530373734 +38353261653139636336363638323334326237323234386333643537393637376535646365323138 +33613962363635393034353035336138623461666163316663386532633666393734386438303166 +64633464333938333465383335353434363232663738313866336139633235336133666131396561 +63363030663862336437386663666332633438303337656533386166663532663138633833633036 +61393838356566353466643234386365386563346664323039393030623965303135383436643366 +35313639323530396337653661616562393732633764313965356262633431326562356230613630 +35323564613061393766636363373865633333376137336530643865306531613331356231366534 +33303037306565356337303532653134653437343534646439383765323536353332346331613664 +64383566333639363132303661303333313537346632343266393932396565633463643565643362 +36376431373165656331626533336632303563653938663232626138646464373535656536613664 +61336539356134623136643363616431343030393930633231323563393164643666316636613130 +37653238623762336132303637613364363931633062623662636566376439393962356434663630 +31646165343631623937653533393563316537333532383733316666343761303031363038356139 +36346162623764386665393338373661636432626536363837303164326438323663653865303662 +31363139353233313137333537663063373233396265376261356138383034633535306663643061 +66323730373062633937336431333065643636656130323434393533353334343136393135616139 +36636533666262663932303035636637306533303732666364616231316433303431646534353236 +30653930613962383465373366336439396437336333353966353637623334373063396537303965 +64376635373534666364633764613664643339376438316430386361326534326533376463386434 +33633839396134313939393137343531353534623337313535393266386165633663376537383732 +33336364393263393465313738653732653535656633616636313934313763616438323634613834 +35363131383931626466353062336632636132663063336131386137653539383666653832383232 +39633166333635653638323139663462353535656566326463373064613232343265376638343966 +39343436616234626435636634623030616561376164323439316363333464363535323862383636 +31626536633231343465656638646632376136383333623731616331393862313334313264313737 +36386334346131373739316662376162613139623763343632643066613164376533623436393635 +35316135666139363033356131343763396331666334386133323532336231376566656362653461 +64323462393132633033353035306262656366613961306436663865343830343432633238356336 +61633739333962353135346430646564306361393865336435663839303336623766653263316537 +64343364346530346630633034363838363062396166386165313261363262326334303636313932 +33306361373031316261366661356333343565616662396136383335333234373663303330323335 +37663231323933333262386361326537386135396539306232333061343537323830303461323137 +30663063383333396133633739636531646132346137643930653438396433383536386233326436 +36303666353337643164356261363563653536316534363961653831643462626536323131653730 +30336662633933363562623838663038633138353961386638343362316536386430363966366465 +36333034623337653932333765313366343131343331386132616437346332663864313735396131 +30306439363862386339663638613337343135353864393134393939626162616366613663323636 +31623633373138316430373535623661353637633866626334663832653763633839393039326634 +61333263666536313964653534343638356435646336386539356538316363656237343532373134 +65366466396231333739666663396638653066646333376462396164383061386339663333373731 +38366332356433326437343339393061306437326538333463353335356162643462373831663661 +34333864623333326430393663633365356331346630343932336338363731383861303932303266 +39353162626130353361303437643233633532653161333330316437356539646136636632343662 +39666365633466356638316430623635333763623961313561636237656263326430393937303663 +63393665393563326565326363346632326138643939363062623066373833326138303734656633 +64333632366236313663633438653665383162363835356430643864643733333663373630393733 +30326537303331356162316331346636663339616435333861323666396338393738373162646231 +63343837363766316231306164643262373863643130336530303032373238363738353738636266 +34383564346437373265363236373136633235626331353834643064303761303630646130383564 +32663065343236393865666261343937326534666339656163653237393139333161633431643435 +38663232336236663438623364306332306236303961663635333932303631306264613131323639 +30383362663638303966623461313366613032373637623034393530313066613363366663396231 +66356236353335383865383437356137353437623166346461363732373632316536626235353631 +66626565663062666138336239346435343463323762306434626565366232356163393432383131 +39616538633437356430613330626339313132383362373231343061346139316164343538363436 +31323162653433383534663864373434353332613836373033646365633936356536636565346161 +64396635333338343639363834323065633833616437303839333963373966366361616231353038 +61613065376638393366373037346263306438303531623665383137613362306230323532633763 +30333665373133376337626635343036626430653761313865333963633036663266346166383337 +38383437613365633935356431643931376633656461363264646364663034396238336230636161 +31353361303263643662623234626266386133396531303830656532393736383638383833383833 +66376236623737313539396430346561623965613263353239383861313330336533613539626134 +32636532356638386338333831393363316538303132316662363235663661306434366334343333 +62633835616566333961663561643161306461356230323532356562623838393963633731333731 +63663631666133623362306633353630353932373363623332326263663432643634376232643435 +64643261336161626534616466373630663964653138633735636133333334323766386633336232 +62653239313366386136313939383537653733396534346139663138633734306234643662626563 +62383538323733333265333536633531663737336337613138643166383134336331653161626436 +39346563363163303731303133323737353830343837623431633566363030613863646262613962 +62356265363765323239653930356339663336333866396263343631666332366139333430306664 +35306634616230636438 diff --git a/host_vars/dev-postgres02-01.yml b/host_vars/dev-postgres02-01.yml new file mode 100644 index 0000000..a47a98a --- /dev/null +++ b/host_vars/dev-postgres02-01.yml @@ -0,0 +1,3 @@ +--- +server_type: "master" +lvm_volume_encryption: true diff --git a/host_vars/dev-postgres02-02.yml b/host_vars/dev-postgres02-02.yml new file mode 100644 index 0000000..4d95a12 --- /dev/null +++ b/host_vars/dev-postgres02-02.yml @@ -0,0 +1,3 @@ +--- +server_type: "slave" +lvm_volume_encryption: true diff --git a/patchday.yml b/patchday.yml index faf786a..2b7dc59 100644 --- a/patchday.yml +++ b/patchday.yml @@ -125,7 +125,7 @@ - hosts: postgres serial: 1 - become: yes + become: true tasks: - name: "Smardigo Patchday: stop service(s)" ansible.builtin.systemd: @@ -134,10 +134,10 @@ - name: "Smardigo Patchday: update pkgs" ansible.builtin.apt: - upgrade: yes - update_cache: yes - autoremove: yes - autoclean: yes + upgrade: true + update_cache: true + autoremove: true + autoclean: true - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" ansible.builtin.reboot: @@ -145,7 +145,7 @@ reboot_timeout: 300 - name: "Smardigo Patchday: wait_for host after reboot" - become: no + become: false delegate_to: localhost ansible.builtin.wait_for: delay: 15 @@ -154,22 +154,46 @@ host: '{{ stage_server_ip }}' search_regex: OpenSSH - - name: "Smardigo Patchday: start services" + - name: "Open and mount LUKS encrypted LVM for datadir" + ansible.builtin.include_role: + name: lvm_with_hetzner_volumes + when: lvm_volume_encryption | bool + vars: + lvm_with_hetzner_volumes__volprefix: "postgres_datadir" + lvm_with_hetzner_volumes__volsize: "{{ postgres_pgdatadir_lvm_hcloudvol_size }}" + lvm_with_hetzner_volumes__volcount: "{{ postgres_pgdatadir_lvm_hcloudvol_count }}" + lvm_with_hetzner_volumes__mountpath: "{{ postgres_pgdatadir_lvm_hcloudvol_mountpath }}" + lvm_with_hetzner_volumes__passphrase: "{{ postgres_volume_encryption_passphrase }}" + + - name: "Open and mount LUKS encrypted LVM for backupdir" + ansible.builtin.include_role: + name: lvm_with_hetzner_volumes + when: lvm_volume_encryption | bool and server_type == "slave" + vars: + lvm_with_hetzner_volumes__volprefix: "postgres-backup" + lvm_with_hetzner_volumes__volsize: "{{ postgres_backup_volume_size }}" + lvm_with_hetzner_volumes__volcount: "{{ postgres_backup_volume_count }}" + lvm_with_hetzner_volumes__mountpath: "{{ backup_directory }}" + lvm_with_hetzner_volumes__passphrase: "{{ postgres_volume_encryption_passphrase }}" + + - name: "Smardigo Patchday: restart services" ansible.builtin.systemd: name: postgresql - state: started + state: restarted # wait_for cannot be used anymore due to enabled SSL encryption for postgres connections in DEV-382 - name: "Smardigo Patchday: check if postgres is listing on net internal ip address" - become: no + become: false community.postgresql.postgresql_ping: port: 5432 ssl_mode: require login_host: '{{ stage_private_server_ip }}' register: check_postgres - ignore_errors: yes + ignore_errors: true - name: "Smardigo Patchday: error-handling - ensure postgres started and check listing on net internal ip address" + when: + - not check_postgres.is_available block: - name: "Smardigo Patchday: error-handling - ensure service(s) started" @@ -178,7 +202,7 @@ state: started - name: "Smardigo Patchday: error-handling - check if postgres is listing on net internal ip address" - become: no + become: false community.postgresql.postgresql_ping: port: 5432 ssl_mode: require @@ -194,7 +218,7 @@ host: localhost port: 25 to: '{{ devops_email_address }}' - subject: "patchday( {{ lookup('pipe','date +%Y-%m-%d_%H:%M') }} ) problem report for {{ inventory_hostname }}" + subject: "patchday( {{ lookup('pipe', 'date +%Y-%m-%d_%H:%M') }} ) problem report for {{ inventory_hostname }}" body: | Dear Sir or Madam, @@ -206,27 +230,24 @@ your automation-bofh - when: - - not check_postgres.is_available - # due to bloody dependencies in SMA application startup, iam must be available during startup # => patching IAM service outsourced in separate part to make sure that is up and running - hosts: iam,keycloak serial: 10 - become: yes + become: true tasks: - name: "Smardigo Patchday: update pkgs" ansible.builtin.apt: - upgrade: yes - update_cache: yes - autoremove: yes - autoclean: yes + upgrade: true + update_cache: true + autoremove: true + autoclean: true - name: "Smardigo Patchday: find docker_compose.yml files" ansible.builtin.find: paths: '{{ service_base_path }}' pattern: 'docker*.yml' - recurse: yes + recurse: true register: docker_compose_services - name: "Smardigo Patchday: shutdown services" @@ -241,7 +262,7 @@ reboot_timeout: 300 - name: "Smardigo Patchday: wait_for host after reboot" - become: no + become: false delegate_to: localhost ansible.builtin.wait_for: delay: 15 diff --git a/roles/lvm_with_hetzner_volumes/tasks/_remove_hetzner_volumes.yml b/roles/lvm_with_hetzner_volumes/tasks/_remove_hetzner_volumes.yml index ddbae62..1edda21 100644 --- a/roles/lvm_with_hetzner_volumes/tasks/_remove_hetzner_volumes.yml +++ b/roles/lvm_with_hetzner_volumes/tasks/_remove_hetzner_volumes.yml @@ -1,7 +1,6 @@ --- - - name: "Getting all hcloud volumes for {{ inventory_hostname }}" - hcloud_volume_info: + hetzner.hcloud.hcloud_volume_info: api_token: "{{ hetzner_authentication_ansible }}" label_selector: "stage={{ stage }},bound_on={{ inventory_hostname }}" register: hcloud_volumes_found @@ -9,31 +8,27 @@ become: false - name: "Getting all hcloud volumes for {{ inventory_hostname }}" - debug: + ansible.builtin.debug: msg: "{{ hcloud_volumes_found }}" - name: "Setting all hcloud volumes for {{ inventory_hostname }}" - set_fact: + ansible.builtin.set_fact: hcloud_volumes: "{{ hcloud_volumes_found.hcloud_volume_info | json_query(jmesquery) }}" vars: jmesquery: "[*].name" -- name: "remove delete protection for all hcloud volumes for {{ inventory_hostname }}" - hcloud_volume: +- name: "Remove delete protection for all hcloud volumes for {{ inventory_hostname }}" + hetzner.hcloud.hcloud_volume: api_token: "{{ hetzner_authentication_ansible }}" name: "{{ item }}" - delete_protection: no + delete_protection: false delegate_to: 127.0.0.1 loop: "{{ hcloud_volumes }}" - name: "Delete all hcloud volumes for {{ inventory_hostname }}" - hcloud_volume: + hetzner.hcloud.hcloud_volume: api_token: "{{ hetzner_authentication_ansible }}" name: "{{ item }}" state: absent delegate_to: 127.0.0.1 loop: "{{ hcloud_volumes }}" - - - - diff --git a/roles/lvm_with_hetzner_volumes/tasks/main.yml b/roles/lvm_with_hetzner_volumes/tasks/main.yml index ea82e72..cd3c4fc 100644 --- a/roles/lvm_with_hetzner_volumes/tasks/main.yml +++ b/roles/lvm_with_hetzner_volumes/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: "Creating some hcloud volumes for LVM purpose" - hcloud_volume: + hetzner.hcloud.hcloud_volume: api_token: "{{ hetzner_authentication_ansible }}" name: "{{ lvm_with_hetzner_volumes__volprefix }}-{{ inventory_hostname }}--vol{{ item }}" server: "{{ inventory_hostname }}" @@ -11,69 +11,85 @@ vol_no: "{{ item | string }}" size: "{{ lvm_with_hetzner_volumes__volsize }}" state: present - delete_protection: yes - loop: "{{ range(1, lvm_with_hetzner_volumes__volcount+1) | list }}" + delete_protection: true + loop: "{{ range(1, lvm_with_hetzner_volumes__volcount + 1) | list }}" register: created_volume delegate_to: localhost become: false - name: "Getting all hcloud volumes for {{ inventory_hostname }}" - hcloud_volume_info: + hetzner.hcloud.hcloud_volume_info: api_token: "{{ hetzner_authentication_ansible }}" label_selector: "stage={{ stage }},used_for={{ lvm_with_hetzner_volumes__volprefix }},bound_on={{ inventory_hostname }}" register: hcloud_volumes_found delegate_to: localhost become: false -- name: "Getting all hcloud volumes for {{ inventory_hostname }}" - debug: - msg: "{{ hcloud_volumes_found }}" - - name: "Setting LVM related VARs" - set_fact: - pvs: "{{ hcloud_volumes_found.hcloud_volume_info | json_query(jmesquery) }}" + ansible.builtin.set_fact: + pv_paths: "{{ hcloud_volumes_found.hcloud_volume_info | map(attribute='linux_device') | list }}" + pv_names: "{{ hcloud_volumes_found.hcloud_volume_info | map(attribute='name') | list }}" vg_name: "vg.{{ lvm_with_hetzner_volumes__volprefix }}" lv_name: "lv.{{ lvm_with_hetzner_volumes__volprefix }}" - vars: - jmesquery: "[*].linux_device" + encrypt_volumes: "{{ lvm_volume_encryption | bool }}" + when: hcloud_volumes_found.hcloud_volume_info | length > 0 + +- name: Manage LUKS container(s) + when: encrypt_volumes + block: + - name: Create LUKS container(s) + community.crypto.luks_device: + device: "{{ item.0 }}" + state: "present" + name: "{{ item.1 }}" + passphrase: "{{ lvm_with_hetzner_volumes__passphrase }}" + loop: "{{ pv_paths | zip(pv_names) | list }}" -- name: "Creating a volume group on top of all found hcloud volumes" + - name: Open LUKS container(s) + community.crypto.luks_device: + device: "{{ item.0 }}" + state: "opened" + name: "{{ item.1 }}" + passphrase: "{{ lvm_with_hetzner_volumes__passphrase }}" + loop: "{{ pv_paths | zip(pv_names) | list }}" + +- name: "Creating a volume group on top of all found volumes" community.general.lvg: vg: "{{ vg_name }}" - pvs: "{{ pvs }}" - pvresize: yes + pvs: "{{ pv_names | map('regex_replace', '^(.*)$', '/dev/mapper/\\1') if encrypt_volumes else pv_paths }}" + pvresize: true register: create_vg + when: hcloud_volumes_found.hcloud_volume_info | length > 0 - name: "Create logical volume" # noqa no-handler community.general.lvol: vg: "{{ vg_name }}" - lv: "{{ lv_name }}" - size: '100%PVS' - when: - - create_vg.changed + lv: "{{ lv_name }}" + size: "100%PVS" + when: create_vg.changed - name: "Format volume" - filesystem: + community.general.system.filesystem: fstype: ext4 dev: "/dev/{{ vg_name }}/{{ lv_name }}" - name: "Resize volume" # noqa no-handler - filesystem: + community.general.system.filesystem: fstype: ext4 dev: "/dev/{{ vg_name }}/{{ lv_name }}" - resizefs: yes + resizefs: true when: - create_vg.changed # set noqa linter 'tag' due to unknown file permissions/ownership for mount path ; # must be set in role etc in which this role will be called!!! - name: "Ensure mountpath exists without setting permission/ownership" # noqa risky-file-permissions - file: + ansible.builtin.file: path: "{{ lvm_with_hetzner_volumes__mountpath }}" state: directory - name: "Mount created LVM volume" - mount: + ansible.posix.mount: path: "{{ lvm_with_hetzner_volumes__mountpath }}" src: "/dev/{{ vg_name }}/{{ lv_name }}" fstype: ext4 diff --git a/roles/postgres/tasks/base-requirements.yml b/roles/postgres/tasks/base-requirements.yml index c8f1709..099621a 100644 --- a/roles/postgres/tasks/base-requirements.yml +++ b/roles/postgres/tasks/base-requirements.yml @@ -8,7 +8,7 @@ name: postgres uid: 2000 group: postgres - home: '{{ postgres_homedir }}' + home: "{{ postgres_homedir }}" system: true shell: /bin/bash @@ -45,14 +45,14 @@ cache_valid_time: 900 state: present loop: - - 'postgresql-{{ default_postgres_version }}' + - "postgresql-{{ default_postgres_version }}" - python3-psycopg2 - name: "Set vars" set_fact: - cert_private_key: '{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-key.pem' - cert_public_key: '{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-crt.pem' - ca_cert: '{{ postgres_homedir }}/ca-certificate.pem' + cert_private_key: "{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-key.pem" + cert_public_key: "{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-crt.pem" + ca_cert: "{{ postgres_homedir }}/ca-certificate.pem" - name: "Include role for self-signed CA" include_role: @@ -63,15 +63,15 @@ name: selfsigned_ca tasks_from: _create_cert vars: - selfsigned_ca_cert_private_key: '{{ cert_private_key }}' + selfsigned_ca_cert_private_key: "{{ cert_private_key }}" selfsigned_ca_cert_private_key_group: postgres - selfsigned_ca_cert_public_key: '{{ cert_public_key }}' - selfsigned_ca_cacert: '{{ ca_cert }}' + selfsigned_ca_cert_public_key: "{{ cert_public_key }}" + selfsigned_ca_cacert: "{{ ca_cert }}" selfsigned_ca_cert_subject: - CN: '{{ inventory_hostname }}.{{ domain }}' + CN: "{{ inventory_hostname }}.{{ domain }}" selfsigned_ca_cert_altnames: - - 'DNS:{{ inventory_hostname }}.{{ domain }}' - - 'DNS:{{ inventory_hostname }}' + - "DNS:{{ inventory_hostname }}.{{ domain }}" + - "DNS:{{ inventory_hostname }}" # selfsigned_ca_trigger_handler: restart postgres - name: "ASSERT: stage_private_server_ip" @@ -86,9 +86,16 @@ regex: "{{ item.regex }}" line: "{{ item.line }}" path: /etc/postgresql/{{ default_postgres_version }}/main/postgresql.conf - loop: '{{ postgres_config }}' + loop: "{{ postgres_config }}" notify: restart postgres +- name: "Updating pg_hba.conf entry for trusted admin user 'postgres'" + lineinfile: + state: "{{ database_state }}" + regex: "^hostssl[ ]+postgres[ ]+postgres" + line: "hostssl postgres postgres {{ stage_private_server_ip }}/32 trust" + path: /etc/postgresql/{{ default_postgres_version }}/main/pg_hba.conf + - name: "Creating archive directory if necessary" file: state: directory @@ -99,7 +106,7 @@ - name: "Install postgres exporter via include_task" include_tasks: install_postgres_exporter.yml - args: + args: apply: tags: - postgres-exporter diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml index 484e3f5..6972712 100644 --- a/roles/postgres/tasks/main.yml +++ b/roles/postgres/tasks/main.yml @@ -1,5 +1,4 @@ --- - ### tags: - name: "Create/Resize LVM for datadir" @@ -10,6 +9,7 @@ lvm_with_hetzner_volumes__volsize: "{{ postgres_pgdatadir_lvm_hcloudvol_size }}" lvm_with_hetzner_volumes__volcount: "{{ postgres_pgdatadir_lvm_hcloudvol_count }}" lvm_with_hetzner_volumes__mountpath: "{{ postgres_pgdatadir_lvm_hcloudvol_mountpath }}" + lvm_with_hetzner_volumes__passphrase: "{{ postgres_volume_encryption_passphrase }}" # Minimal requirements for postgres - name: Include Base Requirements @@ -36,6 +36,7 @@ lvm_with_hetzner_volumes__volsize: "{{ postgres_backup_volume_size }}" lvm_with_hetzner_volumes__volcount: "{{ postgres_backup_volume_count }}" lvm_with_hetzner_volumes__mountpath: "{{ backup_directory }}" + lvm_with_hetzner_volumes__passphrase: "{{ postgres_volume_encryption_passphrase }}" when: server_type == "slave" args: apply: @@ -43,5 +44,3 @@ - backup-requirements tags: - backup-requirements - - diff --git a/roles/postgres/tasks/master-requirements.yml b/roles/postgres/tasks/master-requirements.yml index c346348..20f93b2 100644 --- a/roles/postgres/tasks/master-requirements.yml +++ b/roles/postgres/tasks/master-requirements.yml @@ -1,24 +1,31 @@ --- -- name: Check role exists # noqa no-changed-when - become: yes +- name: Check role exists + become: true become_user: postgres - shell: "/usr/bin/psql -Atc \"SELECT count(rolname) FROM pg_roles where rolname='replicator'\"" # noqa command-instead-of-shell + ansible.builtin.shell: '/usr/bin/psql -Atc "SELECT count(rolname) FROM pg_roles where rolname=''replicator''"' # noqa command-instead-of-shell register: role_check + ignore_errors: true # noqa ignore-errors no-changed-when - name: Create role if necessary - become: yes + become: true become_user: postgres - shell: "/usr/bin/psql -c 'CREATE ROLE replicator WITH REPLICATION LOGIN;'" + ansible.builtin.shell: "/usr/bin/psql -c 'CREATE ROLE replicator WITH REPLICATION LOGIN;'" when: role_check.stdout == "0" + register: cmd_ret + changed_when: cmd_ret.rc != 0 -- name: Change password with scram-sha-256! for replicator and set password # noqa no-changed-when - become: yes +- name: Change password with scram-sha-256! for replicator and set password + become: true become_user: postgres - shell: "/usr/bin/psql -c \"set password_encryption = 'scram-sha-256';ALTER ROLE replicator WITH PASSWORD '{{ postgres_replicator_user_password }}';\"" + ansible.builtin.shell: >- + /usr/bin/psql -c "SET password_encryption = 'scram-sha-256'; + ALTER ROLE replicator WITH PASSWORD '{{ postgres_replicator_user_password }}'" + register: cmd_ret + changed_when: cmd_ret.rc != 0 - name: Setup pg_hba.conf for replicator user - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^host[ ]+replication[ ]+replicator" line: "host replication replicator {{ shared_service_pg_slave_ip }}/32 trust" @@ -26,7 +33,7 @@ register: pg_hba_conf_replicator - name: Set 'wal_level = replica' for master postgresql instance - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^wal_level" line: "wal_level = replica" @@ -34,7 +41,7 @@ register: wal_level - name: Set 'max_wal_senders = 10' for master postgresql instance - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^max_wal_senders" line: "max_wal_senders = 10" @@ -42,7 +49,7 @@ register: max_wal_senders - name: Set 'archive_mode = on' for master postgresql instance - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^archive_mode" line: "archive_mode = on" @@ -50,7 +57,7 @@ register: archive_mode - name: Set 'archive_command = cp -f %p /postgresql/replication/%f' for master postgresql instance - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^archive_command" line: "archive_command = 'cp -f %p /postgresql/replication/%f'" @@ -58,76 +65,109 @@ register: archive_command - name: Set 'wal_keep_size = 16' for master postgresql instance - lineinfile: + ansible.builtin.lineinfile: state: present regex: "^wal_keep_size" line: "wal_keep_size = 16" path: /etc/postgresql/{{ default_postgres_version }}/main/postgresql.conf register: wal_keep_size -- name: Install nfs-server - apt: name=nfs-kernel-server state=present update_cache=yes cache_valid_time=900 - -- name: Install nfs-common - apt: name=nfs-common state=present update_cache=yes cache_valid_time=900 +- name: Install nfs packages + ansible.builtin.apt: + name: + - nfs-kernel-server + - nfs-common + state: present + update_cache: true + cache_valid_time: 900 - name: Create nfs share for archive - lineinfile: + ansible.builtin.lineinfile: path: /etc/exports regex: "^/postgresql/replication" line: "/postgresql/replication/ {{ shared_service_pg_slave_ip }}/32(rw,crossmnt,root_squash,no_subtree_check,sync)" state: present register: nfsshare_archive_check +- name: Get service facts + ansible.builtin.service_facts: + +- name: Check existence of necessary services + vars: + services: + - name: postgresql.service + - name: nfs-kernel-server.service + block: + - name: "Check state of {{ item.name }}" + ansible.builtin.fail: + msg: "{{ item.name }} is not present on this system, why? It should have been there!" + when: ansible_facts.services[item.name] is not defined + loop: "{{ services }}" + - name: Restart nfs-server if necessary # noqa no-handler - service: + ansible.builtin.service: name: nfs-kernel-server state: restarted - when: nfsshare_archive_check.changed + when: nfsshare_archive_check.changed or + ansible_facts.services["nfs-kernel-server.service"].state != "active" - name: Restart postgres if necessary # noqa no-handler - service: + ansible.builtin.service: name: postgresql state: restarted - when: - pg_hba_conf_replicator.changed or + when: pg_hba_conf_replicator.changed or wal_level.changed or archive_mode.changed or archive_command.changed or max_wal_senders.changed or - wal_keep_size.changed + wal_keep_size.changed or + ansible_facts.services["postgresql.service"].state != "active" -- name: Create extension pgcrypto for template1 # noqa no-changed-when - become: yes +- name: Create extension pgcrypto for template1 + become: true become_user: postgres - shell: "/usr/bin/psql template1 -c \"create extension if not exists pgcrypto;\"" + ansible.builtin.shell: '/usr/bin/psql template1 -c "create extension if not exists pgcrypto;"' + ignore_errors: true # noqa ignore-errors no-changed-when -- name: Check database replication_cron exists # noqa no-changed-when - become: yes +- name: Check database replication_cron exists + become: true become_user: postgres - shell: "/usr/bin/psql -Atc \"SELECT count(*) FROM pg_database WHERE datname = 'replication_cron'\"" + ansible.builtin.shell: '/usr/bin/psql -Atc "SELECT count(*) FROM pg_database WHERE datname = ''replication_cron''"' register: database_replication_check + ignore_errors: true # noqa ignore-errors no-changed-when - name: Create replication_cron update database - become: yes + become: true become_user: postgres - shell: "/usr/bin/psql -c \"CREATE DATABASE replication_cron;\"" + ansible.builtin.shell: '/usr/bin/psql -c "CREATE DATABASE replication_cron;"' when: database_replication_check.stdout == "0" + ignore_errors: true # noqa ignore-errors no-changed-when -- name: Create replication update schema # noqa no-changed-when - become: yes +- name: Create replication update schema + become: true become_user: postgres - shell: "/usr/bin/psql replication_cron -c \"CREATE SCHEMA IF NOT EXISTS replication_cron;\"" + ansible.builtin.shell: '/usr/bin/psql replication_cron -c "CREATE SCHEMA IF NOT EXISTS replication_cron;"' + ignore_errors: true # noqa ignore-errors no-changed-when -- name: Create replication update table # noqa no-changed-when - become: yes +- name: Create replication update table + become: true become_user: postgres - shell: "/usr/bin/psql replication_cron -c \"CREATE TABLE IF NOT EXISTS replication_cron.replication_cron (dt timestamp);\"" + ansible.builtin.shell: '/usr/bin/psql replication_cron -c "CREATE TABLE IF NOT EXISTS replication_cron.replication_cron (dt timestamp);"' + ignore_errors: true # noqa ignore-errors no-changed-when -- name: Create dummy update data # noqa no-changed-when - become: yes +- name: Create dummy update data + become: true become_user: postgres - shell: "/usr/bin/psql replication_cron -c \"INSERT INTO replication_cron.replication_cron SELECT now() WHERE NOT EXISTS (SELECT 1 from replication_cron.replication_cron);\"" + ansible.builtin.shell: >- + /usr/bin/psql replication_cron -c + "INSERT INTO replication_cron.replication_cron + SELECT NOW() + WHERE NOT EXISTS + (SELECT 1 + FROM replication_cron.replication_cron)" + register: cmd_ret + changed_when: cmd_ret.rc != 0 + ignore_errors: true # noqa command-instead-of-shell - name: Ensure a cron runs every 5 minutes and update replication check table" ansible.builtin.cron: @@ -135,42 +175,46 @@ minute: "*/5" job: su - postgres -c "/usr/bin/psql replication_cron -c \"UPDATE replication_cron.replication_cron SET dt=now();\"" -- name: Check replication slot exists # noqa no-changed-when - become: yes +- name: Check replication slot exists + become: true become_user: postgres - shell: "/usr/bin/psql -Atc \"select count(*) from pg_replication_slots where slot_name='pgstandby1'\"" + ansible.builtin.shell: '/usr/bin/psql -Atc "select count(*) from pg_replication_slots where slot_name=''pgstandby1''"' register: replication_slot_check + ignore_errors: true # noqa ignore-errors no-changed-when - name: Create replication-slot - become: yes + become: true become_user: postgres - shell: "/usr/bin/psql -Atc \"SELECT pg_create_physical_replication_slot('pgstandby1');\"" + ansible.builtin.shell: '/usr/bin/psql -Atc "SELECT pg_create_physical_replication_slot(''pgstandby1'');"' when: replication_slot_check.stdout == "0" + register: cmd_ret + changed_when: cmd_ret.rc != 0 + ignore_errors: true # noqa command-instead-of-shell # only needed in case of install from scratch - name: "Ensure test db stuff" - block: - - name: "Copy testdb.sql to ensure test DB" - copy: - src: '{{ item }}' - dest: '/tmp/{{ item }}' - mode: '0444' - owner: postgres - group: postgres - loop: - - testdb.sql - - - name: "Ensure test DB" - become: yes - become_user: postgres - community.postgresql.postgresql_db: - name: dummytestdb - - - name: "Ensure content for test DB" - become: yes - become_user: postgres - community.postgresql.postgresql_db: - name: dummytestdb - state: restore - target: /tmp/testdb.sql when: postgres_ensure_testdb | default(False) + block: + - name: "Copy testdb.sql to ensure test DB" + ansible.builtin.copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + mode: "0444" + owner: postgres + group: postgres + loop: + - testdb.sql + + - name: "Ensure test DB" + become: true + become_user: postgres + community.postgresql.postgresql_db: + name: dummytestdb + + - name: "Ensure content for test DB" + become: true + become_user: postgres + community.postgresql.postgresql_db: + name: dummytestdb + state: restore + target: /tmp/testdb.sql diff --git a/roles/postgres/tasks/slave-requirements.yml b/roles/postgres/tasks/slave-requirements.yml index e9dbc34..3a171f1 100644 --- a/roles/postgres/tasks/slave-requirements.yml +++ b/roles/postgres/tasks/slave-requirements.yml @@ -1,5 +1,4 @@ --- - - name: Install nfs-common apt: name=nfs-common state=present update_cache=yes cache_valid_time=900 @@ -29,6 +28,11 @@ line: "{{ shared_service_pg_master_ip }} db-backups" state: present +- name: Wait for nfsd is up and running + wait_for: + host: "{{ shared_service_pg_master_ip }}" + port: 2049 + - name: Mount replication NFS volume ansible.posix.mount: src: db-master:/postgresql/replication @@ -47,6 +51,11 @@ state: absent path: /var/lib/postgresql/{{ default_postgres_version }}/main/ +- name: Wait for db-master is up and running + wait_for: + host: "{{ shared_service_pg_master_ip }}" + port: 5432 + - name: Sync data from db-master # noqa command-instead-of-shell no-changed-when become: yes become_user: postgres diff --git a/stage-dev b/stage-dev index 2786056..1459755 100644 --- a/stage-dev +++ b/stage-dev @@ -43,10 +43,18 @@ dev-pgadmin4-01 [postfix] dev-mail-01 -[postgres] +[postgres01] dev-postgres-01 dev-postgres-02 +#[postgres02] +#dev-postgres02-01 +#dev-postgres02-02 + +[postgres:children] +postgres01 +#postgres02 + [prometheus] dev-prometheus-01