From be4a9c3f5c4dad3ae86cd4478ecbe21f6ad4fd75 Mon Sep 17 00:00:00 2001 From: "Ketelsen, Sven" Date: Fri, 13 Aug 2021 10:48:06 +0000 Subject: [PATCH] SMARCH-63: Feat: Anlegen/Konfigurieren einer Datenbank auf dem DB-Server (DEV) --- create-database-cluster.yml | 11 +- create-database-container.yml | 1 + create-realm.yml | 1 + create-server.yml | 1 + create-service.yml | 1 + group_vars/all/plain.yml | 7 +- group_vars/all/vault.yml | 160 +++++++++--------- group_vars/connect/plain.yml | 5 +- group_vars/postgres/plain.yml | 10 +- group_vars/stage_dev/plain.yml | 22 ++- host_vars/dev-elastic-stack-01.yml | 7 - host_vars/dev-elastic-stack-02.yml | 7 - host_vars/dev-elastic-stack-03.yml | 7 - host_vars/dev-postgres-01.yml | 3 + host_vars/dev-postgres-02.yml | 3 + roles/connect-postgres/defaults/main.yml | 6 + roles/connect-postgres/handlers/main.yml | 1 + roles/connect-postgres/meta/main.yml | 1 + roles/connect-postgres/tasks/main.yml | 35 ++++ roles/connect-postgres/vars/main.yml | 1 + roles/connect/vars/main.yml | 3 +- roles/pgadmin4/defaults/main.yml | 4 + roles/pgadmin4/handlers/main.yml | 1 + roles/pgadmin4/meta/main.yml | 1 + roles/pgadmin4/tasks/main.yml | 74 ++++++++ roles/pgadmin4/vars/main.yml | 50 ++++++ roles/postgres/defaults/main.yml | 2 +- .../{postgres-acls.yml => _postgres-acls.yml} | 60 +++---- roles/postgres/tasks/main.yml | 33 ---- roles/prometheus/tasks/main.yml | 4 + roles/prometheus/vars/main.yml | 2 +- setup.yml | 11 ++ .../provisioning/script/ansible-start.groovy | 5 +- .../config/prometheus/prometheus.yml.j2 | 21 +++ 34 files changed, 375 insertions(+), 186 deletions(-) create mode 100644 host_vars/dev-postgres-01.yml create mode 100644 host_vars/dev-postgres-02.yml create mode 100644 roles/connect-postgres/defaults/main.yml create mode 100644 roles/connect-postgres/handlers/main.yml create mode 100644 roles/connect-postgres/meta/main.yml create mode 100644 roles/connect-postgres/tasks/main.yml create mode 100644 roles/connect-postgres/vars/main.yml create mode 100644 roles/pgadmin4/defaults/main.yml create mode 100644 roles/pgadmin4/handlers/main.yml create mode 100644 roles/pgadmin4/meta/main.yml create mode 100644 roles/pgadmin4/tasks/main.yml create mode 100644 roles/pgadmin4/vars/main.yml rename roles/postgres/tasks/{postgres-acls.yml => _postgres-acls.yml} (64%) diff --git a/create-database-cluster.yml b/create-database-cluster.yml index 9952af0..52d7e5c 100644 --- a/create-database-cluster.yml +++ b/create-database-cluster.yml @@ -1,5 +1,9 @@ --- +# creates postgres databases on shared service postgres server +# - executed on stage specific postgres server: {{ stage }}-postgres-01 +# - creates databases to work with connect: {{ connect_postgres_database_name }} + # Parameters: # playbook inventory # stage := the type of the stage (e.g. dev, int, qa, prod) @@ -31,11 +35,11 @@ tasks: - name: Add hosts add_host: - name: "{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-{{ '%02d' | format(item|int) }}" + name: "{{ stage }}-postgres-01" groups: - "stage_{{ stage }}" - "{{ cluster_service }}" - with_sequence: start=1 end={{ cluster_size | default(1) }} + changed_when: False ############################################################# # Setup databases for created inventory @@ -76,8 +80,7 @@ - debug roles: - - role: postgres - when: "'postgres' in group_names" + - role: connect-postgres ############################################################# # Sending smardigo management message to process diff --git a/create-database-container.yml b/create-database-container.yml index fb1fc6f..10951aa 100644 --- a/create-database-container.yml +++ b/create-database-container.yml @@ -36,6 +36,7 @@ - "stage_{{ stage }}" - "{{ cluster_service }}" with_sequence: start=1 end={{ cluster_size | default(1) }} + changed_when: False ############################################################# # Setup databases for created inventory diff --git a/create-realm.yml b/create-realm.yml index e3fb284..f826dab 100644 --- a/create-realm.yml +++ b/create-realm.yml @@ -39,6 +39,7 @@ - "stage_{{ stage }}" - "{{ cluster_service }}" with_sequence: start=1 end={{ cluster_size | default(1) }} + changed_when: False ############################################################# # Setup realms for created inventory diff --git a/create-server.yml b/create-server.yml index 4137564..4fd44dc 100644 --- a/create-server.yml +++ b/create-server.yml @@ -36,6 +36,7 @@ - "stage_{{ stage }}" - "{{ cluster_service }}" with_sequence: start=1 end={{ cluster_size | default(1) }} + changed_when: False ############################################################# # Provisioning servers for created inventory diff --git a/create-service.yml b/create-service.yml index ab03afe..00c94fa 100644 --- a/create-service.yml +++ b/create-service.yml @@ -24,6 +24,7 @@ - "stage_{{ stage }}" - "{{ cluster_service }}" with_sequence: start=1 end={{ cluster_count | default(1) }} + changed_when: False ############################################################# # Setup services for created inventory diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index e47f0ef..bdeba00 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -45,11 +45,13 @@ sudo_group: "{{ sudo_groups default_plattform_users: - 'nobody' - 'vagrant' + - 'ansible' + - 'elastic' + - 'postgres' - 'administrator' - '{{ admin_user }}' smardigo_plattform_users: - - 'ansible' - 'peter.heise' - 'sven.ketelsen' - 'claus.paetow' @@ -119,6 +121,9 @@ hetzner_ssh_keys: #hetzner_authentication_token: "< see vault >" #digitalocean_authentication_token: "< see vault >" +#pgadmin4_admin_username: "< see vault >" +#pgadmin4_admin_password: "< see vault >" + #elastic_admin_username: "< see vault >" #elastic_admin_password: "< see vault >" diff --git a/group_vars/all/vault.yml b/group_vars/all/vault.yml index 97e7683..16f16bf 100644 --- a/group_vars/all/vault.yml +++ b/group_vars/all/vault.yml @@ -1,77 +1,85 @@ $ANSIBLE_VAULT;1.1;AES256 -61653336363762373661306133636238666261646334366539636532303830303534396432363337 -3639643337313965333439663937303937613763326439650a663863646335643432336661326661 -66316266366538393161373334383765363965393035633562383039396631623962663666313762 -6637373237623762620a656261633562366136616562363564346630646236613463316533643663 -39633261643163373535333332356364393465363636316130663030376232303732653339343964 -31303666366264613265336337376433316435396537613937623039623733373663623739363232 -32376435613161643030363739323735346436316436343133373338363232663464363134626663 -35313734666137626335313334666433666562373461653435653862333432343462376430356332 -33373336613766666131646633303834353832383461386362336164643562656562343061383133 -30646430346232303836653032373235633166313233383839386565343266323634306531326366 -30326136613363643766613262393930653864383037656630623434376661646231376335633530 -39303438633230623063643832646164646261343938313334313762323538666530313066666530 -35616664656437383735633831623032316137373133353639666161643962663730613034346635 -39393133346334633461663061386165616562383531316636343734636331613764386339613861 -61623138616334313763356530323533366362366135383962653061353732613937303337663533 -36646435383466316461616563613837393230396232616237663635353534336237663765376433 -35353136373130323564393634313332373936396161316634333362663431643031366435383230 -37343333343636373964363834356337656266616337313035326338353930356561616431383139 -36663966386366656338343561363764313563656238326536383363646363613431626463306263 -39353430623763333935373935306365336165383432633733353536363462373633643663363363 -63613435363763353731653162353538336137303863363136613633306533326134616230383331 -63303561393564353031643639353032386362653733303766303235373365616237353734373765 -65336233343865313961363132616563626664353338666536636463313963326163613864333964 -38306333313938373238373832313131316162316132653861646337333465656433633530303565 -33646565373166353630343031613166623762653737303335316430386334363565663538316433 -37323661366262363735373134623162316630653132666565626632396664326132323439373233 -37613738643463616135626236306665313961376261356466643462396466303234356536393739 -39396338373438316165343566376466343935306237656463356537623230313663306232623562 -31333662316438393131376333323934646165373232393034613763373166333335313634393630 -32303666356338326231613033363863353536373930653132656331653334366539376436393864 -39343863656539306365383563666631376264623862346562333635653934376531356331303737 -30313665343337333762383861373835303339663335346236646361333234393435623663353437 -31396535333163373762333431306563393962303537633763363638616136323231326162386664 -35636161383361323561373236363461616464346535336535666632626435376637326239333738 -33633336643133373461616631613530623636316536633631643034326230376633616563316565 -33363962313762386432613536646334313961383739663339623362336564353339396233326133 -30366464646466636435303531616430326561643630343138633935663966383234356662376637 -39323839376163626633353963633161636432656663343962626337616665323432303735366530 -63383964636665646432646337353634313434343564613139333363303863396663303263363631 -30313366633361386636643134336566326565373861666334623563646133666261333235626537 -37323666333566343430626137313563303336376437366238663234653566653437363633366436 -37633961616337656565323739336435393533663862643632373933633733626165363630353430 -30343536613164343461333464616638313139306535313032303364653765363462646235656230 -64346436613231353334356266613562663032386564633534373332306563656365346534633137 -65346533353035383864643739646631333938613461373234323234383063373138353230323862 -30313062346663336633666231666530613134386236653362396331356330336234393365393062 -34396534393232643764623932633963306461373065643865376239393262616635303230616161 -62643161656237373531373934346461653062636136383533376436383462316538373064656433 -33306638393637316134636663633331643134333532366666316536366233333839356632393038 -38353034326563366235623865616164323530303037306432623761616131633361303237323736 -62346439623131343964323665316438656366376638333362336365393334316132393633376637 -66373037353339383232346437656239383336643436323462663933653031653439393433653265 -38393162356235313733376466633535313463346638613365333235656631343436376138333361 -65653264323861623739613262333365386530373063623332323965383938653465646261633139 -33663235373261653039636630666138363034373662373561386630386633393433386436623530 -30663863646666633865393431356633326634393364396166636639343931333564626437373066 -66383639313738613234626361323564383739396335356632313961376137363735346566653639 -62663838646133653465333139613734356336613762393032363131333961336335633237373535 -31363661636538336435306537343963323938376561643962313730336562323932346334376133 -37396662356165653861346230316335653765666136323930346531363334353035636661333339 -34343836373437666139376662643239616435396438663634653136323836386563366264313332 -37386637396663366332333931613436396337303664653437363931383164323038373036383661 -34313532333732656238643061323735393362343036633039653462626239323739643433343637 -35316331613937643931653264663230643939613136373065343334356131303665656561326437 -36323936373463356665356334363161343130393130646133393339316337333834333163663034 -63623039633266323736623364386665616630633133353138326566363830616233343464363338 -31366165303633636234623363336234376162346233623137386264656236653039373862396337 -35633034306138626464653637626664376638353837303236633134396264383731363931643432 -38623931346432333632653863643362346561653136623361643362616339326138383863386561 -66396435373137396263393132663834393631653461333235303135653962326235646336653463 -63663435313632613337633861653463626363643531356664663735326462353936336539663632 -31356638623939376137663666616664333037653236373135306333316362346361666165633764 -32313532343566363962663166356465303631353764333931323337636261373562666364643266 -64303636623633323736336262636164336164366433343633356362323866306561383966383134 -32633438343238373537373936376365616634396336633634356262393630383333323030663932 -37356437396635356636 +32396563623961633038643633316466663939653264373037343730653639663938343630346562 +6230363031386138656435346433656535303531613761630a663435613837343536316133323038 +65653162656237623039633464666462376436383562303366323464373961386533343832333862 +6366353533313863640a313331343431393530313264633930346332666265393530653739643933 +32353866373066383335623863663534396564333534646161313162663330363532633834346136 +36646636663738373635623630623637376430636464326265386239313037666536636664623030 +66303238306236666333326232303162343037626466396235343364613931663264633738303734 +35636535663436633062616362313766333564653566303062653065633131393939363565643465 +31663962363337636164373131353431396138366563633430656330613339633632303837383133 +31386430393037643465333136653536303438626337653163616662323234326532626132313334 +65386433376162613663303537353334383637363634306266646331636138666436356138363635 +35643136646261653433343437326534343166393662353263366666623764366366356331306564 +65626336333563646237376139636438306362616131333534316635393532343061346132333338 +37646331393833303937313564316236353636663631313639616132396563316133363863346330 +30353239303431376639663766613131376238353033313536303431363830356437663831623964 +65656334373736646438323530653234343931626234393661323339393661333863363535653365 +36333932656266393235303238656237323065366132303463666431623462633838393561303132 +34346633336561663831633033633236623333383965393065653136306431366438643633646434 +31326535333131336239666431613833363834616630383231353264613337303034303265653732 +62343336626630376633336466663734636166626137323464633732386236386437636533356233 +66333666303032326132306535376366653233663761653830306163666137643764333635343763 +32346635373731333737326230323233623434323236303566383363303966383036373531313634 +66396339633865326138386535343932653639393962663634313964636332666130366464323666 +62303462303139346262313333396431326637663736363430323363643535363763326239616235 +35303066363461383534323835663635363665356236356438383731306632633330343332356562 +66333034623237383331643135666336306133646433643164373330383638653134356161663563 +38323137633165306634313863353934663030636231653239616261363732393563316634343265 +61653430376131653962616461383563343837313930613464313966353338323833316461323461 +39393237663762343238396534333565393938363835346238643732376161326235303330616663 +37646463333962353930386130303036653834383166623065383530323435333163323330623262 +65316432393661346365373263336333396164643763663438646366393863396632633932376337 +34366264306636356464663734613963646264393364626330633936623364366231643233343263 +64633631633365333364303836633334616534623339336532356431616339663939303037386532 +61343162393337346430353035343136336333363734653538613163626166356131363237313561 +66326338366561323633373137313263323138313037623130383166346266396339373863386636 +32646430663431636363323737653934306337666263656137636632373239363762373038613761 +64613133383635393534356432653636633961613939363938646630633966363339336266386638 +65326231663631633636633439653931656562336361333836623836653030306363336665616166 +39303835636366323562343966326464616239386330346237626261376364613638633835613762 +35333336323430666464303838663330326163343132653036353030303034356561353138633665 +30363931316337303166316363323632376236663861636439633231333263383263373034366538 +30313534613530653635663237383265363164663264363538386235323337393963396236393739 +61343764633737356531646130623136376434366337316264366132346664306561323432623261 +65373632333362313436346633303233623536343738623336336363343638393533636238626530 +32373737623733633639326166656536663332383063386430653334616436663730383037326233 +66656536303063613866393637373839353462386537306236363234613539346438383366333061 +61656138356562666136353665613130626661353562316239303735643966333866663031383834 +63616331656263333034353232656638343438646537393635663836313361333062663634613263 +31656139346565303139386531353935643462373531316363613537363164633437663738363439 +66613530323137393538616366373262353130323930313363656536393265623839333036316135 +34656263616231316439396333386438313533323664653433323463373638653635336130366635 +30633564383439333939633165633235366466396664386532386535303561316538396237616339 +32616137626634373263303165346664646365663866643663383834626337353362353433306232 +32323838313039363633336565663135396262376339396633663364393839346661376534303538 +61636530666631363037323130343862336266613633626631633931633038626363343334646462 +66366263306364656365303263643161666535353534366637373061633866376239303131613564 +37303539396536346635633564383136363666613138336134333561386261616136633534616531 +37303031396633396237316134393963393636666530656635303364383263346561656134643639 +39386132376438396532353361383263646336396662643662326561373339356665663364356535 +30373031333663653665366635616634613262663536643631646637326235613030346161313963 +34656239323130353238616263306361396335303139636237383938343364623331666136333639 +33663962303731396133383431363230303934653937303536396366343161626462393263616666 +63383134383666313133313337303931326366653134643561613234616362313431633639653663 +39613063353738643661613066373730353766626233363033636237366463656361343038663538 +37343866666639303063383561396664386266343736663266373433333535653134613362303664 +61666663373864626266363363376338313036326535663632383030316239313466306433333934 +32663934363765346161333465326662386562356538393339626534393262336639333261666330 +36356564346536366166626536663831653731643730313765343830316565396135646164326337 +66316236343966623234613862366630383734616232386135623265636464333661663636373739 +62636532313365363734323938333230633031356334386264333663623237326565656666343536 +30643535383434663137633135616363613935353638646561323062366430393064383030656431 +34376137333164613263343937343939616366383038306135303231393766373963653434623038 +30663134626330366231343565383330363666353466656233346531633936376265373965613633 +65326638363537323534616537323932316635663233383536613239366232626661346233336435 +64633336616463376561306130353763303763643432316437366562323837373161656531356465 +65393766333336366263353934623432626261343633343761326535353233383166336263656137 +36353132626430616663336566663865356139366238613130326337363735623861363835633735 +33363263666361343066643438376638663232383435303966643737623530663339616534343565 +31303162386663356432303336356466383866356138313537626262653336306563663161383863 +32376439633137313137346636646635623132653632656634623936643833633835316563666438 +35643637633861346361633533333131646364613935626132336331316633326435366433613762 +66666134366433623036633666303733313535663030386439386138623365396166646434643932 +39333665656363316265636530316430646364643565643238653537393930633130323935326463 +35653065656131613836 diff --git a/group_vars/connect/plain.yml b/group_vars/connect/plain.yml index 0e7c61a..185dbae 100644 --- a/group_vars/connect/plain.yml +++ b/group_vars/connect/plain.yml @@ -36,8 +36,9 @@ current_realm_clients: [ } ] -connect_postgres_database: "connect-postgres" -connect_postgres_admin_username: "connect-postgres-admin" +connect_postgres_host: "{{ shared_service_pg_master_hostname }}" +connect_postgres_database_name: "{{ stage }}_{{ tenant_id }}_{{ cluster_name }}_{{ cluster_service }}" +connect_postgres_admin_username: "{{ connect_postgres_database_name }}" connect_postgres_admin_password: "connect-postgres-admin" connect_elastic_host: "dev-elastic-stack-01-elastic" diff --git a/group_vars/postgres/plain.yml b/group_vars/postgres/plain.yml index d9bc925..c29af59 100644 --- a/group_vars/postgres/plain.yml +++ b/group_vars/postgres/plain.yml @@ -1,10 +1,8 @@ --- -hetzner_server_labels: "stage={{ stage }} service=postgres" hetzner_server_type: cpx11 +hetzner_server_labels: "stage={{ stage }} service=postgres" + +postgres_acls: [] -postgres_acls: - - name: smardigo - user: smardigo - password: smardigo - trusted_cidr_entry: 10.0.0.0/16 +pgadmin4_enabled: true diff --git a/group_vars/stage_dev/plain.yml b/group_vars/stage_dev/plain.yml index 045e7b3..e6ec831 100644 --- a/group_vars/stage_dev/plain.yml +++ b/group_vars/stage_dev/plain.yml @@ -4,9 +4,8 @@ stage: "dev" alertmanager_channel_smardigo: "#monitoring-qa" -shared_service_netword: "10.0.0.0/16" - # TODO read configuration with hetzner rest api +shared_service_network: "10.0.0.0/16" shared_service_elastic_01: "10.0.0.2" shared_service_elastic_02: "10.0.0.3" shared_service_elastic_03: "10.0.0.4" @@ -18,15 +17,14 @@ shared_service_mail_ip: "10.0.0.8" shared_service_pg_master_ip: "10.0.0.17" shared_service_pg_slave_ip: "10.0.0.18" -shared_service_awx_hostname: dev-awx-01.smardigo.digital -shared_service_docker_registry_hostname: dev-docker-registry-01.smardigo.digital +shared_service_awx_hostname: "dev-awx-01.smardigo.digital" +shared_service_docker_registry_hostname: "dev-docker-registry-01.smardigo.digital" shared_service_iam_hostname: "dev-iam-01.smardigo.digital" shared_service_keycloak_hostname: "dev-keycloak-01.smardigo.digital" shared_service_mail_hostname: "dev-mail-01.smardigo.digital" shared_service_pg_master_hostname: "dev-postgres-01.smardigo.digital" shared_service_pg_slave_hostname: "dev-postgres-02.smardigo.digital" - shared_service_hosts: [ { ip: "127.0.1.1", @@ -127,6 +125,10 @@ prometheus_extra_hosts: [ } ] connect_extra_hosts: [ + { + hostname: "{{ shared_service_pg_master_hostname }}", + ip: "{{ shared_service_pg_master_ip }}", + }, { hostname: dev-elastic-stack-01-elastic, ip: "{{ shared_service_elastic_01 }}", @@ -172,6 +174,16 @@ iam_extra_hosts: [ ip: "{{ shared_service_mail_ip }}", } ] +pgadmin_extra_hosts: [ + { + hostname: "{{ shared_service_pg_master_hostname }}", + ip: "{{ shared_service_pg_master_ip }}", + }, + { + hostname: "{{ shared_service_pg_slave_hostname }}", + ip: "{{ shared_service_pg_slave_ip }}", + } +] smardigo_management_url: "https://dev-management-smardigo-01.smardigo.digital/api/v1/scopes/{{ scope_id }}/processes/{{ process_instance_id }}/messages" smardigo_management_token: "eyJlbmMiOiJBMTI4Q0JDLUhTMjU2IiwiYWxnIjoiZGlyIn0..JgixZcmsSyvovabQvREAjw.Fk7aNYwOjzMhLCqF_9unl5yrWTey26z4scZBeVZjhpE.fnovrqn0MUjM_TA8zVhXdQ" diff --git a/host_vars/dev-elastic-stack-01.yml b/host_vars/dev-elastic-stack-01.yml index 9a23d20..b0bce9a 100644 --- a/host_vars/dev-elastic-stack-01.yml +++ b/host_vars/dev-elastic-stack-01.yml @@ -3,10 +3,3 @@ hetzner_server_labels: "stage={{ stage }} service=elastic" hetzner_server_type: cx31 - -smardigo_plattform_users: - - 'ansible' - - 'elastic' - - 'peter.heise' - - 'sven.ketelsen' - - 'claus.paetow' diff --git a/host_vars/dev-elastic-stack-02.yml b/host_vars/dev-elastic-stack-02.yml index 9a23d20..b0bce9a 100644 --- a/host_vars/dev-elastic-stack-02.yml +++ b/host_vars/dev-elastic-stack-02.yml @@ -3,10 +3,3 @@ hetzner_server_labels: "stage={{ stage }} service=elastic" hetzner_server_type: cx31 - -smardigo_plattform_users: - - 'ansible' - - 'elastic' - - 'peter.heise' - - 'sven.ketelsen' - - 'claus.paetow' diff --git a/host_vars/dev-elastic-stack-03.yml b/host_vars/dev-elastic-stack-03.yml index 9a23d20..b0bce9a 100644 --- a/host_vars/dev-elastic-stack-03.yml +++ b/host_vars/dev-elastic-stack-03.yml @@ -3,10 +3,3 @@ hetzner_server_labels: "stage={{ stage }} service=elastic" hetzner_server_type: cx31 - -smardigo_plattform_users: - - 'ansible' - - 'elastic' - - 'peter.heise' - - 'sven.ketelsen' - - 'claus.paetow' diff --git a/host_vars/dev-postgres-01.yml b/host_vars/dev-postgres-01.yml new file mode 100644 index 0000000..ae5d8f1 --- /dev/null +++ b/host_vars/dev-postgres-01.yml @@ -0,0 +1,3 @@ +--- + +server_type: "master" diff --git a/host_vars/dev-postgres-02.yml b/host_vars/dev-postgres-02.yml new file mode 100644 index 0000000..39e2488 --- /dev/null +++ b/host_vars/dev-postgres-02.yml @@ -0,0 +1,3 @@ +--- + +server_type: "slave" diff --git a/roles/connect-postgres/defaults/main.yml b/roles/connect-postgres/defaults/main.yml new file mode 100644 index 0000000..f4b7875 --- /dev/null +++ b/roles/connect-postgres/defaults/main.yml @@ -0,0 +1,6 @@ +--- + +postgres_acls: + - name: "{{ connect_postgres_database_name }}" + password: "{{ connect_postgres_admin_password }}" + trusted_cidr_entry: "{{ shared_service_network }}" diff --git a/roles/connect-postgres/handlers/main.yml b/roles/connect-postgres/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect-postgres/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/connect-postgres/meta/main.yml b/roles/connect-postgres/meta/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect-postgres/meta/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/connect-postgres/tasks/main.yml b/roles/connect-postgres/tasks/main.yml new file mode 100644 index 0000000..2a50475 --- /dev/null +++ b/roles/connect-postgres/tasks/main.yml @@ -0,0 +1,35 @@ +--- + +### tags: +### update_deployment + +- name: "Send mattermost message" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-start.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages + +- name: "Setup postgres for {{ service_name }}" + include_role: + name: postgres + tasks_from: _postgres-acls + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-end.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages diff --git a/roles/connect-postgres/vars/main.yml b/roles/connect-postgres/vars/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect-postgres/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/connect/vars/main.yml b/roles/connect/vars/main.yml index a639711..e56bc3f 100644 --- a/roles/connect/vars/main.yml +++ b/roles/connect/vars/main.yml @@ -1,7 +1,6 @@ --- connect_id: "{{ service_name }}-connect" -connect_postgres_id: "{{ service_name }}-postgres_connect" connect_labels: [ '"traefik.enable=true"', @@ -36,7 +35,7 @@ connect_environment: [ "ADMIN_LOGIN: \"{{ connect_admin_username }}\"", "ADMIN_PASSWORD: \"{{ connect_admin_password }}\"", - "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_id }}:{{ service_port_postgres }}/{{ connect_postgres_database }}\"", + "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_host }}:{{ service_port_postgres }}/{{ connect_postgres_database_name }}\"", "DATASOURCE_USERNAME: \"{{ connect_postgres_admin_username }}\"", "DATASOURCE_PASSWORD: \"{{ connect_postgres_admin_password }}\"", "FILE_WHITELIST_URL: \"{{ connect_whitelist_url | default('') }}\"", diff --git a/roles/pgadmin4/defaults/main.yml b/roles/pgadmin4/defaults/main.yml new file mode 100644 index 0000000..318fb87 --- /dev/null +++ b/roles/pgadmin4/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +pgadmin4_image_name: "dpage/pgadmin4" +pgadmin4_image_version: "5" diff --git a/roles/pgadmin4/handlers/main.yml b/roles/pgadmin4/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/pgadmin4/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/pgadmin4/meta/main.yml b/roles/pgadmin4/meta/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/pgadmin4/meta/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/pgadmin4/tasks/main.yml b/roles/pgadmin4/tasks/main.yml new file mode 100644 index 0000000..a53510f --- /dev/null +++ b/roles/pgadmin4/tasks/main.yml @@ -0,0 +1,74 @@ +--- + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-start.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages + +- name: "Check if pgadmin4/docker-compose.yml exists" + stat: + path: '{{ service_base_path }}/pgadmin4/docker-compose.yml' + register: check_docker_compose_file + +- name: "Stop pgadmin4" + shell: docker-compose down + args: + chdir: '{{ service_base_path }}/pgadmin4' + when: check_docker_compose_file.stat.exists + ignore_errors: yes + +- name: "Deploy docker templates for pgadmin4" + include_role: + name: _deploy + tasks_from: templates + vars: + current_config: "_docker" + current_base_path: "{{ service_base_path }}" + current_destination: "pgadmin4" + current_owner: "{{ docker_owner }}" + current_group: "{{ docker_group }}" + current_docker: "{{ pgadmin4_docker }}" + +- name: "Deploy service templates for pgadmin4" + include_role: + name: _deploy + tasks_from: templates + vars: + current_config: "pgadmin4" + current_base_path: "{{ service_base_path }}" + current_destination: "pgadmin4" + current_owner: "{{ docker_owner }}" + current_group: "{{ docker_group }}" + +- name: "Update pgadmin4" + shell: docker-compose pull + args: + chdir: '{{ service_base_path }}/pgadmin4' + tags: + - update_deployment + +- name: "Start pgadmin4" + shell: docker-compose up -d + args: + chdir: '{{ service_base_path }}/pgadmin4' + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-end.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages diff --git a/roles/pgadmin4/vars/main.yml b/roles/pgadmin4/vars/main.yml new file mode 100644 index 0000000..aaf946d --- /dev/null +++ b/roles/pgadmin4/vars/main.yml @@ -0,0 +1,50 @@ +--- + +pgadmin_id: "{{ service_name }}-pgadmin" + +pgadmin4_docker: { + networks: [ + { + name: front-tier, + external: true, + }, + { + name: back-tier, + external: true, + }, + ], + volumes: [ + { + name: pgadmin_data + }, + ], + services: [ + { + name: "{{ pgadmin_id }}", + image_name: "{{ pgadmin4_image_name }}", + image_version: "{{ pgadmin4_image_version }}", + labels: [ + '"traefik.enable=true"', + '"traefik.http.routers.{{ pgadmin_id }}.service={{ pgadmin_id }}"', + '"traefik.http.routers.{{ pgadmin_id }}.rule=Host(`{{ stage_server_url_host }}`)"', + '"traefik.http.routers.{{ pgadmin_id }}.entrypoints=admin-postgres"', + '"traefik.http.routers.{{ pgadmin_id }}.tls=true"', + '"traefik.http.routers.{{ pgadmin_id }}.tls.certresolver=letsencrypt"', + '"traefik.http.services.{{ pgadmin_id }}.loadbalancer.server.port={{ http_port }}"', + ], + environment: [ + "PGADMIN_DEFAULT_EMAIL: \"{{ pgadmin4_admin_username }}\"", + "PGADMIN_DEFAULT_PASSWORD: \"{{ pgadmin4_admin_password }}\"", + "PGADMIN_CONFIG_CONSOLE_LOG_LEVEL: \"20\"", + ], + volumes: [ + '"pgadmin_data:/var/lib/pgadmin"', + ], + networks: [ + '"front-tier"', + '"back-tier"', + ], + extra_hosts: "{{ pgadmin_extra_hosts | default([]) }}", + } + ], +} \ No newline at end of file diff --git a/roles/postgres/defaults/main.yml b/roles/postgres/defaults/main.yml index 4625689..23ac2d5 100644 --- a/roles/postgres/defaults/main.yml +++ b/roles/postgres/defaults/main.yml @@ -8,4 +8,4 @@ default_shared_buffers: 256MB default_master_ip: "{{ shared_service_pg_master_ip }}" default_slave_ip: "{{ shared_service_pg_slave_ip }}" -default_private_network: "{{ shared_service_netword }}" +default_private_network: "{{ shared_service_network }}" diff --git a/roles/postgres/tasks/postgres-acls.yml b/roles/postgres/tasks/_postgres-acls.yml similarity index 64% rename from roles/postgres/tasks/postgres-acls.yml rename to roles/postgres/tasks/_postgres-acls.yml index 3fcc393..e162496 100644 --- a/roles/postgres/tasks/postgres-acls.yml +++ b/roles/postgres/tasks/_postgres-acls.yml @@ -3,53 +3,51 @@ ### properties: ### postgres_acls: ### - name -### - user ### - password ### - trusted_cidr_entry [default_private_network] -- debug: - msg: "{{ postgres_acls }}" - tags: - - postgres_acls -:1 -- name: "Add pg_hba entries for users/nodes/schemas .." +- name: "Add pg_hba.conf entries for users/nodes/schemas" lineinfile: state: present - regex: '^host[ ]+{{ item.name }}[ ]+{{ item.user }}' - line: 'host {{ item.name }} {{ item.user }} {{ item.trusted_cidr_entry | default(default_private_network) }} md5' + regex: '^host[ ]+{{ item.name }}[ ]+{{ item.name }}' + line: 'host {{ item.name }} {{ item.name }} {{ item.trusted_cidr_entry | default(default_private_network) }} md5' path: /etc/postgresql/{{ default_postgres_version }}/main/pg_hba.conf with_items: "{{ postgres_acls }}" - tags: - - postgres_acls -- name: Check role exists +- name: "Check roles exist" become: yes become_user: postgres - shell: "/usr/bin/psql -Atc \"SELECT count(rolname) FROM pg_roles where rolname='{{ item.user }}'\"" + shell: "/usr/bin/psql -Atc \"SELECT count(rolname) FROM pg_roles where rolname='{{ item.name }}'\"" with_items: "{{ postgres_acls }}" register: role_check changed_when: "role_check.stdout == '0'" - tags: - - postgres_acls -- name: Create role if necessary +- name: "Check roles exist result" + debug: + msg: "{{ role_check }}" + when: + - debug + +- name: "Create roles if necessary" become: yes become_user: postgres shell: "/usr/bin/psql -c 'CREATE ROLE {{ item.item.name }} LOGIN;'" when: item.stdout == '0' with_items: "{{ role_check.results }}" - tags: - - postgres_acls -- name: "check databases exists" +- name: "Check databases exist" become: yes become_user: postgres shell: "/usr/bin/psql -Atc \"SELECT count(*) FROM pg_database WHERE datname = '{{ item.name }}'\"" with_items: "{{ postgres_acls }}" register: database_check changed_when: "database_check.stdout == '0'" - tags: - - postgres_acls + +- name: "Check databases exist result" + debug: + msg: "{{ database_check }}" + when: + - debug - name: "Create Databases if necessary" become: yes @@ -57,23 +55,21 @@ shell: "/usr/bin/psql -c \"CREATE DATABASE {{ item.item.name }};\"" when: item.stdout == '0' with_items: "{{ database_check.results }}" - tags: - - postgres_acls -- name: Change password with scram-sha-256! for users and set password +- name: "Change password with scram-sha-256! for users and set password" become: yes become_user: postgres - shell: "/usr/bin/psql -c \"set password_encryption = 'scram-sha-256';ALTER ROLE {{ item.user }} WITH PASSWORD '{{ item.password }}';\"" + shell: "/usr/bin/psql -c \"set password_encryption = 'scram-sha-256';ALTER ROLE {{ item.name }} WITH PASSWORD '{{ item.password }}';\"" with_items: "{{ postgres_acls }}" - register: role_check - tags: - - postgres_acls - name: "Change owners for databases" become: yes become_user: postgres - shell: "/usr/bin/psql -c \"ALTER DATABASE {{ item.name }} OWNER TO {{ item.user }};\"" + shell: "/usr/bin/psql -c \"ALTER DATABASE {{ item.name }} OWNER TO {{ item.name }};\"" with_items: "{{ postgres_acls }}" - register: role_check - tags: - - postgres_acls + +# TODO: -> factor out as handler +- name: "Reload pg_hba.conf" + become: yes + become_user: postgres + shell: "/usr/bin/psql -c \"SELECT pg_reload_conf();\"" diff --git a/roles/postgres/tasks/main.yml b/roles/postgres/tasks/main.yml index 1626791..38ee488 100644 --- a/roles/postgres/tasks/main.yml +++ b/roles/postgres/tasks/main.yml @@ -1,31 +1,5 @@ --- -### tags: -### postgres_acls - -- name: "Setup master/slave fact for {{ inventory_hostname }}" - set_fact: - server_type: "{% set hostname_splitted = inventory_hostname.split('-') %}{% if hostname_splitted[-1]|int == 1 %}master{% else %}slave{% endif %}" - tags: - - postgres_acls - -- debug: - msg: "Server-Type: {{ server_type }}, Internal-IP {{ default_master_ip }}" - when: - - debug | bool - - server_type == "master" - - tags: - - postgres_acls - -- debug: - msg: "Server-Type: {{ server_type }}, Internal-IP {{ default_slave_ip }}" - when: - - debug | bool - - server_type == "slave" - tags: - - postgres_acls - # Minimal requirements for postgres - name: Include Base Requirements include_tasks: base-requirements.yml @@ -39,10 +13,3 @@ - name: Include Slave Requirements include_tasks: slave-requirements.yml when: server_type == "slave" - -# Apply Postgres ACLs -- name: Include Postgresl ACL Requirements - include_tasks: postgres-acls.yml - when: server_type == "master" - tags: - - postgres_acls diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index f3c29c9..cf99267 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -96,6 +96,10 @@ { name: "mail", label_selector: "stage={{ stage }},service=mail", + }, + { + name: "postgres", + label_selector: "stage={{ stage }},service=postgres", } ] loop_control: diff --git a/roles/prometheus/vars/main.yml b/roles/prometheus/vars/main.yml index ff07088..a71ae02 100644 --- a/roles/prometheus/vars/main.yml +++ b/roles/prometheus/vars/main.yml @@ -33,7 +33,7 @@ prometheus_docker: { services: [ { name: "{{ prometheus_id }}", - image_name: "{{ grafana_image_name }}", + image_name: "{{ prometheus_image_name }}", image_version: "{{ prometheus_image_version }}", labels: [ '"traefik.enable=true"', diff --git a/setup.yml b/setup.yml index a2107f2..48b9ed9 100644 --- a/setup.yml +++ b/setup.yml @@ -13,6 +13,8 @@ msg: "The ansible version has to be at least ({{ ansible_version.full }})" tags: - common + - pgadmin4 + - name: Remove outdated dependencies apt: name: [ @@ -40,6 +42,7 @@ become: false tags: - common + - pgadmin4 - name: "Set current server infos as fact: hetzner_server_infos_json" set_fact: @@ -48,6 +51,7 @@ become: false tags: - common + - pgadmin4 - name: "Read ip address for {{ inventory_hostname }}" set_fact: @@ -58,6 +62,7 @@ become: false tags: - common + - pgadmin4 - name: Print the gathered infos debug: @@ -65,6 +70,7 @@ delegate_to: 127.0.0.1 tags: - common + - pgadmin4 roles: - role: ansible-role-docker @@ -91,3 +97,8 @@ when: traefik_enabled | default(True) tags: - traefik + + - role: pgadmin4 + when: pgadmin4_enabled | default(False) + tags: + - pgadmin4 \ No newline at end of file diff --git a/smardigo/provisioning/script/ansible-start.groovy b/smardigo/provisioning/script/ansible-start.groovy index 953e81f..d22a012 100644 --- a/smardigo/provisioning/script/ansible-start.groovy +++ b/smardigo/provisioning/script/ansible-start.groovy @@ -2,12 +2,13 @@ def env = [ scope_id: contextScopeId, process_instance_id: execution.getProcessInstanceId(), smardigo_management_action: smardigoManagementAction, - cluster_name: tenant.key + '-' + cluster.name, + cluster_name: cluster.name, cluster_service: cluster.service, cluster_size: cluster.size, stage: cluster.stage, current_realm_name: tenant.key, - current_realm_display_name: tenant.name + current_realm_display_name: tenant.name, + tenant_id: tenant.key ] def ansibleCommand= 'ansible-playbook ' + smardigoManagementAction + '.yml --vault-password-file ~/vault-pass' diff --git a/templates/prometheus/config/prometheus/prometheus.yml.j2 b/templates/prometheus/config/prometheus/prometheus.yml.j2 index 90e3d5d..3925018 100644 --- a/templates/prometheus/config/prometheus/prometheus.yml.j2 +++ b/templates/prometheus/config/prometheus/prometheus.yml.j2 @@ -199,3 +199,24 @@ scrape_configs: regex: (.*):.* target_label: instance replacement: $1 + + - job_name: 'postgres-exporter' + scheme: http + metrics_path: '/metrics' + static_configs: + - targets: [ +{% for host in server_group_postgres | default([]) %} + '{{ host }}.{{ domain }}:9187', +{% endfor %} + ] + labels: + env: {{ stage }} + project: smardigo + relabel_configs: + - source_labels: [job] + target_label: job + replacement: 'postgres-exporter' + - source_labels: [__address__] + regex: (.*):.* + target_label: instance + replacement: $1