From 57ff124d00155755e2e008b2b5fe184e72e9b7fc Mon Sep 17 00:00:00 2001 From: Sven Ketelsen Date: Sat, 10 Apr 2021 01:10:01 +0200 Subject: [PATCH] feat: setup prometheus stack --- group_vars/all/plain.yml | 9 + group_vars/all/vault.yml | 71 +- group_vars/stage_dev/plain.yml | 5 + provisioning.yml | 2 +- roles/_digitalocean/tasks/domain.yml | 12 + roles/connect/defaults/main.yml | 133 + roles/connect/handlers/main.yml | 1 + roles/connect/meta/main.yml | 1 + roles/connect/tasks/main.yml | 188 + .../connect/templates/create-new-user.json.j2 | 5 + roles/connect/vars/main.yml | 1 + roles/prometheus/defaults/main.yml | 132 + roles/prometheus/handlers/main.yml | 1 + roles/prometheus/meta/main.yml | 1 + roles/prometheus/tasks/main.yml | 184 + .../templates/postgres-exporter.json.j2 | 16 + roles/prometheus/vars/main.yml | 1 + smardigo.yml | 6 +- stage-dev | 19 +- .../config/alertmanager/config.yml.j2 | 20 + .../alertmanager/templates/notifications.tmpl | 18 + templates/prometheus/config/blackbox.yml.j2 | 12 + .../config/grafana/conf/defaults.ini.j2 | 770 + .../config/grafana/config.monitoring.j2 | 2 + .../dashboards/Elasticsearch.json | 5755 +++++++ .../dashboards/Node_Exporter_Full.json | 13492 ++++++++++++++++ .../provisioning/dashboards/Servers.json | 2230 +++ .../dashboards/Spring_Boot_Statistics.json | 4215 +++++ .../provisioning/dashboards/dashboard.yml | 11 + .../datasources/datasource.yml.j2 | 50 + .../config/prometheus/alert.rules.j2 | 252 + .../blackbox-targets-connect.yml.j2 | 8 + .../config/prometheus/blackbox-targets.yml.j2 | 8 + .../config/prometheus/prometheus.yml.j2 | 92 + templates/traefik/traefik_dynamic.toml.j2 | 2 +- 35 files changed, 27686 insertions(+), 39 deletions(-) create mode 100644 group_vars/stage_dev/plain.yml create mode 100644 roles/connect/defaults/main.yml create mode 100644 roles/connect/handlers/main.yml create mode 100644 roles/connect/meta/main.yml create mode 100644 roles/connect/tasks/main.yml create mode 100644 roles/connect/templates/create-new-user.json.j2 create mode 100644 roles/connect/vars/main.yml create mode 100644 roles/prometheus/defaults/main.yml create mode 100644 roles/prometheus/handlers/main.yml create mode 100644 roles/prometheus/meta/main.yml create mode 100644 roles/prometheus/tasks/main.yml create mode 100644 roles/prometheus/templates/postgres-exporter.json.j2 create mode 100644 roles/prometheus/vars/main.yml create mode 100644 templates/prometheus/config/alertmanager/config.yml.j2 create mode 100644 templates/prometheus/config/alertmanager/templates/notifications.tmpl create mode 100644 templates/prometheus/config/blackbox.yml.j2 create mode 100644 templates/prometheus/config/grafana/conf/defaults.ini.j2 create mode 100644 templates/prometheus/config/grafana/config.monitoring.j2 create mode 100644 templates/prometheus/config/grafana/provisioning/dashboards/Elasticsearch.json create mode 100644 templates/prometheus/config/grafana/provisioning/dashboards/Node_Exporter_Full.json create mode 100644 templates/prometheus/config/grafana/provisioning/dashboards/Servers.json create mode 100644 templates/prometheus/config/grafana/provisioning/dashboards/Spring_Boot_Statistics.json create mode 100644 templates/prometheus/config/grafana/provisioning/dashboards/dashboard.yml create mode 100644 templates/prometheus/config/grafana/provisioning/datasources/datasource.yml.j2 create mode 100644 templates/prometheus/config/prometheus/alert.rules.j2 create mode 100644 templates/prometheus/config/prometheus/blackbox-targets-connect.yml.j2 create mode 100644 templates/prometheus/config/prometheus/blackbox-targets.yml.j2 create mode 100644 templates/prometheus/config/prometheus/prometheus.yml.j2 diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index f5b5312..4a69355 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -106,3 +106,12 @@ hetzner_ssh_keys: #mattermost_hook_smardigo: "< see vault >" #hetzner_authentication_token: "< see vault >" #digitalocean_authentication_token: "< see vault >" + +#grafana_admin_username: "< see vault >" +#grafana_admin_password: "< see vault >" +#grafana_signing_secret: "< see vault >" +#grafana_user_smardigo_login: "< see vault >" +#grafana_user_smardigo_password: "< see vault >" +#grafana_user_guest_login: "< see vault >" +#grafana_user_guest_password: "< see vault >" + diff --git a/group_vars/all/vault.yml b/group_vars/all/vault.yml index 7a87035..9fdacb5 100644 --- a/group_vars/all/vault.yml +++ b/group_vars/all/vault.yml @@ -1,30 +1,43 @@ $ANSIBLE_VAULT;1.1;AES256 -37303936393666363130613561666665623364303361613938633232363532313830316231353935 -3335653838653863326334623161336336653435373664310a363561353430356166373766393836 -31303463343336613665326636643837633362636335343830396438363634656639376463353965 -6131343431653439340a363930643966386237323763613566393235303437376132393865323230 -63616136386133326131656565306233343831343030313935663764353330653231666533333238 -37343938363431333936613065613633396231306365346435623362653437326536663135333532 -38303838663865623737663866633839333835363864616330626335323338626331316263653233 -39343965666461653538303632636564336338313162663833623365396465336232366236383034 -61393839616164386565373930623338323130626330316565383338623634663338366233613963 -32306431383136346263633364626265363737353363396131313461656139393239306537343435 -34643830373965303339373831393465366565663936663061663434633131303033333436346566 -31376330613939356534613534313335653464616436393137396165326262636538656137326532 -63396138383364646339353539363230306461373732333037663862306161333966313462356363 -36656639346238313839623232373738376530633361373565353063323065626634306532626539 -37393038633761396539353233666563316535323965363233656134393365356339626565656134 -63323864653566646531393830396337353139653831343866303039313631613334313431643161 -39383264646566636538626433333937333230383564316437353464613862316532343564316530 -64623935383037326563613533313361333435326166343339616461386437356238376263356334 -33373166613033626130333962366464663262393134623838633937653837653332393061626637 -66653730396436313339616562626230363231303136333235663534626266613831646631633530 -39356263346231373463373761626430376431633135353939656664613632633965323838633362 -65633335643866633530346236653435343565663936376266663862363130303032323436646133 -66643833653363323935353636343430346561346262383436663838636536386638356438663430 -65363262396339323530303663333730313836346565623430633232366138376261393831643137 -37373734333538313566306631373233353364656438323435373265306531396534656265633532 -31643831353931393139663861346234333233353566333435373338393166376333343235303034 -37376565643162303531396566313531643933376663343663636230376338666565323263666539 -65373832373931393265333432313232633536646331633833613561366532363239326538333565 -3134 +64643166313265363734313932666666643238333366393865343132313835666433326366653337 +6135633264613662366233323835663034373761663864350a663161316266653238323332656336 +35323166323062323465623933653538356334666230616339313533613431613234653136386230 +3764333134323538310a353530616532613365326131376664386335336161326638663335326530 +38383166343939316537396332313664313064613561393036353164626566343136623835623237 +62383834376661643537356335646462323962633432336238333033343666326230326639363364 +37666261343733373839613362346166666231663463616436363838626134663861616566663137 +64613666616336303134393161323262386264666232336132376534316461333764363037376531 +63363433363233653839616132373038623436333866326338343130343734323662323137643033 +66313664636565323333646238363339653235346362393435323032656536373838643765313562 +61343437353165666464316135366266623263383033346534666538383566303162393430393761 +39663732636164346633396230336538376236663330323363626132383964323530336338363836 +37333837343836616231643730626134303031376130393431646464646438336334343565326463 +65623237373166303662336362663636333964643866643638666132303862626264353064616163 +39333130396237343431336132383238343535363834356462393430363162643635356363383238 +64626434616365346238626236366232333333366431336436363863316563313462366538366436 +33373264623837346232303131653464376534646438643332626566613735653439646661383536 +38346263346336346137616337363435666466343836656638653638646133303733363365323934 +64393631303163633061393530623535313961313737643638626665303363366439306366373064 +65333563636631373931313837653738356234393036323165663036653565386663313938373430 +34656233326230356262306464323563393066646262613933653032333864326261626330643333 +33636464396263323563626335306432373764353265373833653230333837653363333761666136 +34653035393262623134353361323230323238653034316466663663346462353337613939313238 +61393930373037313266663563386632343262373061333838646531373666383535323065646639 +66326539393061373465613130643761346330623866633263336532663966366339323665323363 +39386539633163653233356439303635646666303662393235316238393934633066373866623230 +34316366656130393738353637626166383563343233383163383639373539626234363265356532 +65303739393637656433656164373934613237336436326630393535633637323865386531646638 +63316365623639373332323366373461393766633662396562306534306466653162633131623131 +65306334656535383137343830323966346337323363343663326438613562643466643666386537 +64326334356561653231346433396439666237626336666239336463333536376130373866343736 +32396233333161313230656461396361626435666664616462363036386636396364636364323966 +31656130323264363862656461616562613934636636373535343333666565626134376266613937 +66393266613635313030356263366235323139663439303861356665333163386334646339613933 +62636338343237376630376364323763383562383462613366393738663237643931636161383631 +30623834383839613531616435613833636662313664323166363935396231643430376330396431 +36366235393933613362303466343433643731363835343862346131343836376132316536633034 +32333263313031313464343562633835323663363965373465633433386566313832346639623232 +63393832643937396130613638623231663137303832616266326461636164393565336537656437 +62393866636233343766633863643532396138636638326531326430613634353564386633343265 +33613930356433356139623830326165323632633039333837623136376661303736356661343364 +3736353162636662646162333934306562626662633931386565 diff --git a/group_vars/stage_dev/plain.yml b/group_vars/stage_dev/plain.yml new file mode 100644 index 0000000..d6d5614 --- /dev/null +++ b/group_vars/stage_dev/plain.yml @@ -0,0 +1,5 @@ +--- + +stage: "dev" + +alertmanager_channel_smardigo: "#monitoring-qa" diff --git a/provisioning.yml b/provisioning.yml index 6c1169f..6a6e925 100644 --- a/provisioning.yml +++ b/provisioning.yml @@ -2,7 +2,7 @@ - name: 'apply setup to {{ host | default("all") }}' hosts: '{{ host | default("all") }}' - serial: "{{ serial_number | default(1) }}" + serial: "{{ serial_number | default(5) }}" gather_facts: no become: no diff --git a/roles/_digitalocean/tasks/domain.yml b/roles/_digitalocean/tasks/domain.yml index 28ee469..1be9817 100644 --- a/roles/_digitalocean/tasks/domain.yml +++ b/roles/_digitalocean/tasks/domain.yml @@ -8,20 +8,28 @@ authorization: Bearer {{ digitalocean_authentication_token }} return_content: yes register: domain_records_response + delegate_to: 127.0.0.1 + become: false - name: Save DNS entry as variable (fact) set_fact: domain_records_response_json: "{{ domain_records_response.json }}" + delegate_to: 127.0.0.1 + become: false - name: Parse DNS entry for {{ record_name }}.{{ domain }} set_fact: domain_record: "{{ domain_records_response_json.domain_records | json_query(jmesquery) | first | default({'name': '-', 'ip': '-'}) }}" vars: jmesquery: '[*].{id: id, name: name, ip: data}' + delegate_to: 127.0.0.1 + become: false - name: Print DNS entry for {{ record_name }}.{{ domain }} debug: msg: "{{ domain_record }}" + delegate_to: 127.0.0.1 + become: false - name: Delete DNS entry for <{{ record_name }}> if necessary uri: @@ -34,6 +42,8 @@ when: domain_record.ip != '-' and record_data != domain_record.ip + delegate_to: 127.0.0.1 + become: false - name: Create DNS entry for <{{ record_name }}> if necessary uri: @@ -55,3 +65,5 @@ domain_record.ip == '-' or record_data != domain_record.ip or record_name != domain_record.name + delegate_to: 127.0.0.1 + become: false diff --git a/roles/connect/defaults/main.yml b/roles/connect/defaults/main.yml new file mode 100644 index 0000000..e7113e1 --- /dev/null +++ b/roles/connect/defaults/main.yml @@ -0,0 +1,133 @@ +--- + +connect_image_name: 'docker.arxes-tolina.de/smardigo/connect-whitelabel-app' + +connect_version: 'latest' +connect_admin_username: "connect-admin" +connect_admin_password: "connect-admin" + +connect_postgres_version: "12" +connect_postgres_database: "connect-postgres" +connect_postgres_admin_username: "connect-postgres-admin" +connect_postgres_admin_password: "connect-postgres-admin" + +connect_mail_properties_base_url: "{{ http_s }}://{{ service_url }}" +connect_mail_properties_base_url_extern: "{{ http_s }}://{{ service_url }}" + +connect_id: "{{ service_name }}-connect" +connect_postgres_id: "{{ service_name }}-postgres-connect" + +connect_labels: [ + '"traefik.enable=true"', + '"traefik.http.routers.{{ connect_id }}.service={{ connect_id }}"', + '"traefik.http.routers.{{ connect_id }}.rule=Host(`{{ service_url }}`)"', + '"traefik.http.routers.{{ connect_id }}.entrypoints=websecure"', + '"traefik.http.routers.{{ connect_id }}.tls=true"', + '"traefik.http.routers.{{ connect_id }}.tls.certresolver=letsencrypt"', + '"traefik.http.services.{{ connect_id }}.loadbalancer.server.port={{ service_port }}"', + + '"traefik.http.routers.{{ connect_id }}-admin.service={{ connect_id }}-admin"', + '"traefik.http.routers.{{ connect_id }}-admin.rule=Host(`{{ service_url }}`)"', + '"traefik.http.routers.{{ connect_id }}-admin.entrypoints=admin-service"', + '"traefik.http.routers.{{ connect_id }}-admin.tls=true"', + '"traefik.http.routers.{{ connect_id }}-admin.tls.certresolver=letsencrypt"', + '"traefik.http.routers.{{ connect_id }}-admin.middlewares={{ connect_id }}-admin-cors"', + '"traefik.http.middlewares.{{ connect_id }}-admin-cors.headers.accesscontrolallowmethods=GET,OPTIONS"', + '"traefik.http.middlewares.{{ connect_id }}-admin-cors.headers.accesscontrolalloworigin=*"', + '"traefik.http.middlewares.{{ connect_id }}-admin-cors.headers.accesscontrolallowheaders=SMA_USER"', + '"traefik.http.services.{{ connect_id }}-admin.loadbalancer.server.port={{ management_port }}"', + + '"traefik.http.routers.{{ connect_id }}-monitor.service=node-exporter"', + '"traefik.http.routers.{{ connect_id }}-monitor.rule=Host(`{{ service_url }}`)"', + '"traefik.http.routers.{{ connect_id }}-monitor.entrypoints=admin-system"', + '"traefik.http.routers.{{ connect_id }}-monitor.tls=true"', + '"traefik.http.routers.{{ connect_id }}-monitor.tls.certresolver=letsencrypt"', +] + +connect_docker: { + networks: [ + { + name: back-tier, + external: true, + }, + { + name: front-tier, + external: true, + }, + ], + volumes: [ + { + name: "{{ connect_postgres_id }}-data" + } + ], + services: [ + { + name: "{{ connect_id }}", + image_name: "{{ connect_image_name }}", + image_version: "{{ connect_version }}", + labels: "{{ connect_labels + ( connect_labels_additional | default([])) }}", + restart: "{{ connect_service_restart | default('always') }}", + environment: [ + "ADMIN_LOGIN: \"{{ connect_admin_username }}\"", + "ADMIN_PASSWORD: \"{{ connect_admin_password }}\"", + + "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_id }}:{{ service_port_postgres }}/{{ connect_postgres_database }}\"", + "DATASOURCE_USERNAME: \"{{ connect_postgres_admin_username }}\"", + "DATASOURCE_PASSWORD: \"{{ connect_postgres_admin_password }}\"", + + "MAIL_PROTOCOL: \"{{ connect_mail_protocol | default('smtp') }}\"", + "MAIL_HOST: \"{{ connect_mail_host | default('smtp.tolina.local') }}\"", + "MAIL_PORT: \"{{ connect_mail_port | default('25') }}\"", + "MAIL_USER: \"{{ connect_mail_user | default('') }}\"", + "MAIL_PASSWORD: \"{{ connect_mail_password | default('') }}\"", + "MAIL_PROPERTIES_SIMULATION: \"{{ connect_mail_properties_simulation | default('true') }}\"", + "MAIL_PROPERTIES_BASE_URL: \"{{ connect_mail_properties_base_url }}\"", + "MAIL_PROPERTIES_BASE_URL_EXTERN: \"{{ connect_mail_properties_base_url_extern }}\"", + "MAIL_PROPERTIES_SENDER: \"{{ connect_mail_properties_sender | default('noreply-connect@arxes-tolina.de') }}\"", + "MAIL_PROPERTIES_SENDER_ALIAS: \"{{ connect_mail_properties_sender_alias | default('noreply-connect') }}\"", + + "AUTH_MODULE: \"{{ connect_auth_module | default('preauth') }}\"", + "OIDC_CLIENT_ID: \"{{ connect_oidc_client_id | default('oidc_config_not_found') }}\"", + "OIDC_CLIENT_SECRET: \"{{ connect_oidc_client_secret | default('oidc_config_not_found') }}\"", + "OIDC_REGISTRATION_ID: \"{{ connect_oidc_registration_id | default('oidc_config_not_found') }}\"", + "OIDC_ISSUER_URI: \"{{ connect_oidc_issuer_uri | default('oidc_config_not_found') }}\"", + "PASSWORD_CHANGE_URL: \"{{ connect_password_change_url | default('') }}\"", + "USER_MANAGEMENT_URL: \"{{ connect_iam_user_management_url | default('') }}\"", + + "IAM_MODULE: \"{{ connect_iam_module | default('embedded') }}\"", + "IAM_CLIENT_ENABLED: \"{{ smardigo_iam_client_enabled | default('false') }}\"", + "EXTERNAL_IAM_SERVER_URL: \"{{ smardigo_iam_client_server_url | default('') }}\"", + + "SMA_API_TOKEN_SECRET: \"{{ connect_api_token_secret | default('') }}\"", + + "SMA_CSRF_TOKEN_NAME: \"{{ connect_csrf_token_name | default('') }}\"", + "SMA_CSRF_TOKEN_VALUE: \"{{ connect_csrf_token_value | default('') }}\"", + + "SPRING_PROFILES_INCLUDE: \"{{ spring_profiles_include | default('swagger') }}\"", + "RIBBON_DISPLAY_ON_ACTIVE_PROFILES: \"{{ ribbon_display_on_active_profiles | default('dev') }}\"", + ], + networks: [ + '"back-tier"', + '"front-tier"', + ], + extra_hosts: "{{ connect_extra_hosts | default([]) }}", + }, + { + name: "{{ connect_postgres_id }}", + image_name: "postgres", + image_version: "{{ connect_postgres_version }}", + environment: [ + 'POSTGRES_DB: "{{ connect_postgres_database }}"', + 'POSTGRES_USER: "{{ connect_postgres_admin_username }}"', + 'POSTGRES_PASSWORD: "{{ connect_postgres_admin_password }}"', + ], + volumes: [ + '"{{ connect_postgres_id }}-data:/var/lib/postgresql/data"', + ], + networks: [ + '"back-tier"', + ], + ports: "{{ connect_postgres_ports | default([]) }}", + }, + ], +} diff --git a/roles/connect/handlers/main.yml b/roles/connect/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/connect/meta/main.yml b/roles/connect/meta/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect/meta/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/connect/tasks/main.yml b/roles/connect/tasks/main.yml new file mode 100644 index 0000000..f1b54c1 --- /dev/null +++ b/roles/connect/tasks/main.yml @@ -0,0 +1,188 @@ +--- + +### tags: +### create_users +### update_deployment + + +- name: "Send mattermost message" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-start.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages + +- name: "Setup DNS configuration for {{ service_name }}" + include_role: + name: _digitalocean + tasks_from: domain + vars: + record_data: "{{ stage_server_ip }}" + record_name: "{{ service_name }}" + +- name: "Setup public DNS configuration for {{ service_name }}" + include_role: + name: _digitalocean + tasks_from: domain + vars: + record_data: "{{ item.ip }}" + record_name: "{{ item.name }}" + loop: "{{ connect_public_dns_entries }}" + when: connect_public_dns_entries is defined + +- name: "Check docker networks" + include_role: + name: _docker + tasks_from: networks + +- name: "Check if {{ service_name }}/docker-compose.yml exists" + stat: + path: '{{ service_base_path }}/{{ service_name }}/docker-compose.yml' + register: check_docker_compose_file + tags: + - update_deployment + +- name: "Stop {{ service_name }}" + shell: docker-compose down + args: + chdir: '{{ service_base_path }}/{{ service_name }}' + when: check_docker_compose_file.stat.exists + ignore_errors: yes + tags: + - update_deployment + +- name: "Deploy service configuration for {{ service_name }}" + include_role: + name: _deploy + tasks_from: configs + vars: + current_config: "connect" + current_base_path: "{{ service_base_path }}" + current_destination: "{{ service_name }}" + current_owner: "{{ docker_owner }}" + current_group: "{{ docker_group }}" + current_docker: "{{ connect_docker }}" + +- name: "Update {{ service_name }}" + shell: docker-compose pull + args: + chdir: '{{ service_base_path }}/{{ service_name }}' + tags: + - update_deployment + +- name: "Start {{ service_name }}" + shell: docker-compose up -d + args: + chdir: '{{ service_base_path }}/{{ service_name }}' + tags: + - update_deployment + +- name: "Update caddy configuration for {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_config + vars: + current_service: "{{ service_name }}" + current_services: [ + { + external: "{{ service_url }}", + internal: "{{ service_name }}-connect:{{ service_port }}", + }, + { + external: "{{ service_url }}:{{ monitor_port_service }}", + internal: "{{ service_name }}-connect:{{ management_port }}", + }, + { + external: "{{ service_url }}:{{ monitor_port_system }}", + internal: "node-exporter:{{ service_port_node_exporter }}", + }, + ] + +- name: "Update public caddy configuration for {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_config + vars: + current_service: "{{ service_name }}-{{ item.name }}" + current_services: [ + { + external: "{{ item.name }}.{{ domain }}", + internal: "{{ service_name }}-connect:{{ service_port }}", + } + ] + loop: "{{ connect_public_dns_entries }}" + when: connect_public_dns_entries is defined + +- name: "Update landing page entries for {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_landing_page + vars: + current_services: [ + { + current_name: "{{ service_name }}", + current_url: "{{ http_s }}://{{ service_url }}", + current_version: "{{ connect_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + management: "{{ http_s }}://{{ service_url }}:{{ monitor_port_service }}/management", + }, + ] + tags: + - update_deployment + +- name: "Update landing page with public entries {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_landing_page + vars: + current_services: [ + { + current_name: "{{ item.name }}", + current_url: "{{ http_s }}://{{ item.name }}.{{ domain }}", + current_version: "{{ connect_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + management: "{{ http_s }}://{{ service_url }}:{{ monitor_port_service }}/management", + }, + ] + loop: "{{ connect_public_dns_entries }}" + when: connect_public_dns_entries is defined + tags: + - update_deployment + +- name: "Update landing page with extra entries for {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_landing_page + vars: + current_services: [ + { + current_name: "{{ item.name }}", + current_url: "{{ item.domain }}", + current_version: "{{ connect_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + management: "{{ http_s }}://{{ service_url }}:{{ monitor_port_service }}/management", + }, + ] + loop: "{{ connect_extra_domain_entries }}" + when: connect_extra_domain_entries is defined + tags: + - update_deployment + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-end.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages diff --git a/roles/connect/templates/create-new-user.json.j2 b/roles/connect/templates/create-new-user.json.j2 new file mode 100644 index 0000000..8021012 --- /dev/null +++ b/roles/connect/templates/create-new-user.json.j2 @@ -0,0 +1,5 @@ +{ + "id": "{{ current_user.userId }}", + "firstName": "{{ current_user.firstName | default('null') }}", + "lastName": "{{ current_user.lastName | default('null') }}" +} diff --git a/roles/connect/vars/main.yml b/roles/connect/vars/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/connect/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml new file mode 100644 index 0000000..395354a --- /dev/null +++ b/roles/prometheus/defaults/main.yml @@ -0,0 +1,132 @@ +--- + +grafana_version: "7.0.5" +prometheus_version: "v2.19.2" +alertmanager_version: "v0.21.0" +blackbox_exporter_version: "v0.17.0" +postgres_exporter_version: "v0.8.0" + +service_port_grafana: 3000 +service_port_prometheus: 9090 +service_port_alertmanager: 9093 +service_port_blackbox_exporter: 9115 +service_port_postgres_exporter: 9187 + +prometheus_id: "{{ service_name }}-prometheus" +alertmanager_id: "{{ service_name }}-alertmanager" +grafana_id: "{{ service_name }}-grafana" + +prometheus_docker: { + networks: [ + { + name: back-tier, + external: true, + }, + { + name: front-tier, + external: true, + }, + ], + volumes: [ + { + name: "{{ prometheus_id }}-data" + }, + { + name: "{{ alertmanager_id }}-data" + }, + { + name: "{{ grafana_id }}-data" + }, + ], + services: [ + { + name: "{{ prometheus_id }}", + image_name: "prom/prometheus", + image_version: "{{ prometheus_version }}", + labels: [ + '"traefik.enable=true"', + '"traefik.http.routers.{{ prometheus_id }}.service={{ prometheus_id }}"', + '"traefik.http.routers.{{ prometheus_id }}.rule=Host(`{{ service_name }}-prometheus.{{ domain }}`)"', + '"traefik.http.routers.{{ prometheus_id }}.entrypoints=websecure"', + '"traefik.http.routers.{{ prometheus_id }}.tls=true"', + '"traefik.http.routers.{{ prometheus_id }}.tls.certresolver=letsencrypt"', + '"traefik.http.services.{{ prometheus_id }}.loadbalancer.server.port={{ service_port_prometheus }}"', + ], + command: [ + '"--config.file=/etc/prometheus/prometheus.yml"', + '"--storage.tsdb.path=/prometheus"', + '"--web.console.libraries=/usr/share/prometheus/console_libraries"', + '"--web.console.templates=/usr/share/prometheus/consoles"', + '"--web.external-url={{ http_s}}://{{ service_name }}-prometheus.{{ domain }}"', + '"--web.enable-lifecycle"', + '"--storage.tsdb.retention.time=30w"', + ], + volumes: [ + '"./config/prometheus/:/etc/prometheus/:ro"', + '"{{ prometheus_id }}-data:/prometheus"', + ], + networks: [ + '"back-tier"', + '"front-tier"', + ], + extra_hosts: "{{ prometheus_extra_hosts | default([]) }}", + }, + { + name: "{{ alertmanager_id }}", + image_name: "prom/alertmanager", + image_version: "{{ alertmanager_version }}", + labels: [ + '"traefik.enable=true"', + '"traefik.http.routers.{{ alertmanager_id }}.service={{ alertmanager_id }}"', + '"traefik.http.routers.{{ alertmanager_id }}.rule=Host(`{{ service_name }}-alertmanager.{{ domain }}`)"', + '"traefik.http.routers.{{ alertmanager_id }}.entrypoints=websecure"', + '"traefik.http.routers.{{ alertmanager_id }}.tls=true"', + '"traefik.http.routers.{{ alertmanager_id }}.tls.certresolver=letsencrypt"', + '"traefik.http.services.{{ alertmanager_id }}.loadbalancer.server.port={{ service_port_alertmanager }}"', + ], + command: [ + '"--config.file=/etc/alertmanager/config.yml"', + '"--storage.path=/alertmanager"', + '"--web.external-url={{ http_s}}://{{ service_name }}-alertmanager.{{ domain }}"', + ], + environment: [ + 'LS_JAVA_OPTS: "-Xmx1G -Xms1G"', + ], + volumes: [ + '"./config/alertmanager/:/etc/alertmanager/:ro"', + '"{{ alertmanager_id }}-data:/alertmanager"', + ], + networks: [ + '"back-tier"', + '"front-tier"', + ], + }, + { + name: "{{ grafana_id }}", + image_name: "grafana/grafana", + image_version: "{{ grafana_version }}", + user: '"472"', + labels: [ + '"traefik.enable=true"', + '"traefik.http.routers.{{ grafana_id }}.service={{ grafana_id }}"', + '"traefik.http.routers.{{ grafana_id }}.rule=Host(`{{ service_name }}-grafana.{{ domain }}`)"', + '"traefik.http.routers.{{ grafana_id }}.entrypoints=websecure"', + '"traefik.http.routers.{{ grafana_id }}.tls=true"', + '"traefik.http.routers.{{ grafana_id }}.tls.certresolver=letsencrypt"', + '"traefik.http.services.{{ grafana_id }}.loadbalancer.server.port={{ service_port_grafana }}"', + ], + volumes: [ + '"./config/grafana/provisioning/:/etc/grafana/provisioning/"', + '"./config/grafana/conf/defaults.ini:/usr/share/grafana/conf/defaults.ini"', + '"{{ grafana_id }}-data:/var/lib/grafana"', + ], + networks: [ + '"back-tier"', + '"front-tier"', + ], + env_file: [ + '"./config/grafana/config.monitoring"', + ], + } + ], +} diff --git a/roles/prometheus/handlers/main.yml b/roles/prometheus/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/prometheus/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/prometheus/meta/main.yml b/roles/prometheus/meta/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/prometheus/meta/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml new file mode 100644 index 0000000..505a2c7 --- /dev/null +++ b/roles/prometheus/tasks/main.yml @@ -0,0 +1,184 @@ +--- + +### tags: +### update_config + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-start.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages + +- name: Gather current server infos + hcloud_server_info: + api_token: "{{ hetzner_authentication_token }}" + register: hetzner_server_infos + delegate_to: 127.0.0.1 + become: false + +- name: Save current server infos as variable (fact) + set_fact: + hetzner_server_infos_json: "{{ hetzner_server_infos.hcloud_server_info }}" + delegate_to: 127.0.0.1 + become: false + +- name: Read ip for {{ inventory_hostname }} + set_fact: + stage_server_ip: "{{ item.ipv4_address }}" + when: item.name == inventory_hostname + with_items: "{{ hetzner_server_infos_json }}" + delegate_to: 127.0.0.1 + become: false + +- name: "Setup DNS configuration for {{ service_name }} prometheus" + include_role: + name: _digitalocean + tasks_from: domain + vars: + record_data: "{{ stage_server_ip }}" + record_name: "{{ service_name }}-prometheus" + +- name: "Setup DNS configuration for {{ service_name }} grafana" + include_role: + name: _digitalocean + tasks_from: domain + vars: + record_data: "{{ stage_server_ip }}" + record_name: "{{ service_name }}-grafana" + +- name: "Setup DNS configuration for {{ service_name }} alertmanager" + include_role: + name: _digitalocean + tasks_from: domain + vars: + record_data: "{{ stage_server_ip }}" + record_name: "{{ service_name }}-alertmanager" + +- name: "Check docker networks" + include_role: + name: _docker + tasks_from: networks + +- name: "Check if {{ service_name }}/docker-compose.yml exists" + stat: + path: '{{ service_base_path }}/{{ service_name }}/docker-compose.yml' + register: check_docker_compose_file + tags: + - update_config + +- name: "Stop {{ service_name }}" + shell: docker-compose down + args: + chdir: '{{ service_base_path }}/{{ service_name }}' + when: check_docker_compose_file.stat.exists + ignore_errors: yes + +- name: "Deploy service configuration for {{ service_name }}" + include_role: + name: _deploy + tasks_from: configs + vars: + current_config: "prometheus" + current_base_path: "{{ service_base_path }}" + current_destination: "{{ service_name }}" + current_owner: "{{ docker_owner }}" + current_group: "{{ docker_group }}" + current_docker: "{{ prometheus_docker }}" + tags: + - update_config + +- name: "Start {{ service_name }}" + shell: | + docker-compose down + docker-compose up -d + args: + chdir: '{{ service_base_path }}/{{ service_name }}' + tags: + - update_config + +- name: "Update landing page for {{ service_name }}" + include_role: + name: _deploy + tasks_from: caddy_landing_page + vars: + current_services: [ + { + current_name: "{{ service_prefix }}prometheus", + current_url: "{{ http_s}}://{{ service_name }}-prometheus.{{ domain }}", + current_version: "{{ prometheus_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + }, + { + current_name: "{{ service_prefix }}grafana", + current_url: "{{ http_s }}://{{ service_name }}-grafana.{{ domain }}", + current_version: "{{ grafana_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + }, + { + current_name: "{{ service_prefix }}alertmanager", + current_url: "{{ http_s }}://{{ service_name }}-alertmanager.{{ domain }}", + current_version: "{{ alertmanager_version }}", + current_date: "{{ ansible_date_time.iso8601 }}", + }, + ] + +- name: "Wait for {{ http_s }}://{{ service_name }}-grafana.{{ domain }}" + uri: + url: "{{ http_s }}://{{ service_name }}-grafana.{{ domain }}/api/admin/stats" + url_username: "{{ grafana_admin_username }}" + url_password: "{{ grafana_admin_password }}" + force_basic_auth: yes + method: GET + status_code: 200 + return_content: yes + register: grafana_stats + until: grafana_stats.status == 200 + retries: 10 + delay: 60 + +- name: Create grafana users + uri: + url: "{{ http_s }}://{{ service_name }}-grafana.{{ domain }}/api/admin/users" + url_username: "{{ grafana_admin_username }}" + url_password: "{{ grafana_admin_password }}" + force_basic_auth: yes + method: POST + status_code: 200 + body_format: json + body: "{\"name\":\"{{ item.name }}\", \"email\":\"{{ item.email }}\", \"login\":\"{{ item.login }}\", \"password\":\"{{ item.password }}\" }" + headers: + Content-Type: application/json + loop: + - { + name: "{{ grafana_user_smardigo_login }}", + email: "smardigo@arxes-tolina.de", + login: "{{ grafana_user_smardigo_login }}", + password: "{{ grafana_user_smardigo_password }}" + } + - { + name: "{{ grafana_user_guest_login }}", + email: "grafana-guest@arxes-tolina.de", + login: "{{ grafana_user_guest_login }}", + password: "{{ grafana_user_guest_password }}" + } + when: grafana_stats.json.users == 1 + +- name: "Send mattermost messsge" + uri: + url: "{{ mattermost_hook_smardigo }}" + method: POST + body: "{{ lookup('template','mattermost-deploy-end.json.j2') }}" + body_format: json + headers: + Content-Type: "application/json" + delegate_to: 127.0.0.1 + become: false + when: + - send_status_messages diff --git a/roles/prometheus/templates/postgres-exporter.json.j2 b/roles/prometheus/templates/postgres-exporter.json.j2 new file mode 100644 index 0000000..ed339c9 --- /dev/null +++ b/roles/prometheus/templates/postgres-exporter.json.j2 @@ -0,0 +1,16 @@ +{% for database in databases %} +{ + name: "{{ service_name }}-postgres-exporter-{{ database.name }}", + image_name: "wrouesnel/postgres_exporter", + image_version: "{{ postgres_exporter_version }}", + environment: [ + 'DATA_SOURCE_URI: "{{ database.uri }}"', + "{{ database.user }}", + "{{ database.pass }}", + ], + networks: [ + '"back-tier"', + '"front-tier"', + ], +} +{% endfor %} \ No newline at end of file diff --git a/roles/prometheus/vars/main.yml b/roles/prometheus/vars/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/prometheus/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/smardigo.yml b/smardigo.yml index 37e3af9..e190a0e 100644 --- a/smardigo.yml +++ b/smardigo.yml @@ -13,5 +13,7 @@ msg: "The ansible version has to be at least ({{ ansible_version.full }})" roles: - - role: docker-registry - when: "'docker_registry' in group_names" + - role: connect + when: "'connect' in group_names" + - role: prometheus + when: "'prometheus' in group_names" diff --git a/stage-dev b/stage-dev index c3aef02..3333477 100644 --- a/stage-dev +++ b/stage-dev @@ -1,20 +1,27 @@ -[hcloud] +[connect] +dev-connect-01 +dev-connect-02 +dev-connect-03 + +[docker_registry] dev-docker-registry-01 + +[elastic] dev-elastic-stack-01 dev-elastic-stack-02 dev-elastic-stack-03 -dev-prometheus-01 - -[docker_registry] -dev-docker-registry-01 [prometheus] dev-prometheus-01 [stage_dev:children] -hcloud +connect docker_registry +elastic prometheus [all:children] stage_dev + +[hcloud:children] +stage_dev diff --git a/templates/prometheus/config/alertmanager/config.yml.j2 b/templates/prometheus/config/alertmanager/config.yml.j2 new file mode 100644 index 0000000..67adf28 --- /dev/null +++ b/templates/prometheus/config/alertmanager/config.yml.j2 @@ -0,0 +1,20 @@ +route: + receiver: smardigo #default + routes: + - match: + project: 'smardigo' + env: '{{ stage }}' + receiver: 'smardigo' + +receivers: + - name: 'smardigo' + slack_configs: + - api_url: '{{ mattermost_hook_smardigo }}' + username: 'prometheus' + channel: '{{ alertmanager_channel_smardigo }}' + title: '{{ '{{' }} template "custom_title" . {{ '}}' }}' + text: '{{ '{{' }} template "custom_slack_message" . {{ '}}' }}' + send_resolved: true + +templates: + - /etc/alertmanager/templates/notifications.tmpl diff --git a/templates/prometheus/config/alertmanager/templates/notifications.tmpl b/templates/prometheus/config/alertmanager/templates/notifications.tmpl new file mode 100644 index 0000000..f5ff165 --- /dev/null +++ b/templates/prometheus/config/alertmanager/templates/notifications.tmpl @@ -0,0 +1,18 @@ +{{ define "__single_message_title" }}{{ range .Alerts.Firing }}{{ .Labels.alertname }} @ {{ .Annotations.identifier }}{{ end }}{{ range .Alerts.Resolved }}{{ .Labels.alertname }} @ {{ .Annotations.identifier }}{{ end }}{{ end }} + +{{ define "custom_title" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if or (and (eq (len .Alerts.Firing) 1) (eq (len .Alerts.Resolved) 0)) (and (eq (len .Alerts.Firing) 0) (eq (len .Alerts.Resolved) 1)) }}{{ template "__single_message_title" . }}{{ end }}{{ end }} + +{{ define "custom_slack_message" }} +{{ if or (and (eq (len .Alerts.Firing) 1) (eq (len .Alerts.Resolved) 0)) (and (eq (len .Alerts.Firing) 0) (eq (len .Alerts.Resolved) 1)) }} +{{ range .Alerts.Firing }}{{ .Annotations.description }}{{ end }}{{ range .Alerts.Resolved }}{{ .Annotations.description }}{{ end }} +{{ else }} +{{ if gt (len .Alerts.Firing) 0 }} +*Alerts Firing:* +{{ range .Alerts.Firing }}- {{ .Annotations.summary }} +{{ end }}{{ end }} +{{ if gt (len .Alerts.Resolved) 0 }} +*Alerts Resolved:* +{{ range .Alerts.Resolved }}- {{ .Annotations.summary }} +{{ end }}{{ end }} +{{ end }} +{{ end }} diff --git a/templates/prometheus/config/blackbox.yml.j2 b/templates/prometheus/config/blackbox.yml.j2 new file mode 100644 index 0000000..c144859 --- /dev/null +++ b/templates/prometheus/config/blackbox.yml.j2 @@ -0,0 +1,12 @@ +modules: + http_200: + prober: http + timeout: 5s + http: + method: GET + fail_if_ssl: false + fail_if_not_ssl: false + tls_config: + insecure_skip_verify: false + preferred_ip_protocol: "ip4" + valid_status_codes: [200] diff --git a/templates/prometheus/config/grafana/conf/defaults.ini.j2 b/templates/prometheus/config/grafana/conf/defaults.ini.j2 new file mode 100644 index 0000000..5b6c965 --- /dev/null +++ b/templates/prometheus/config/grafana/conf/defaults.ini.j2 @@ -0,0 +1,770 @@ +##################### Grafana Configuration Defaults ##################### +# +# Do not modify this file in grafana installs +# + +# possible values : production, development +app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${HOSTNAME} + +#################################### Paths ############################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = data + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = data/log + +# Directory where grafana will automatically scan and look for plugins +plugins = data/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server ############################## +[server] +# Protocol (http, https, h2, socket) +protocol = http + +# The ip address to bind to, empty will bind to all interfaces +http_addr = + +# The http port to use +http_port = 3000 + +# The public facing domain name used to access grafana from a browser +domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +enforce_domain = false + +# The full public facing url +root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +serve_from_sub_path = false + +# Log web requests +router_logging = false + +# the path relative working path +static_root_path = public + +# enable gzip +enable_gzip = false + +# https certs & key file +cert_file = +cert_key = + +# Unix socket path +socket = /tmp/grafana.sock + +#################################### Database ############################ +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url property. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +password = +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +url = + +# Max idle conn setting default is 2 +max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +log_queries = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +ssl_mode = disable + +ca_cert_path = +client_key_path = +client_cert_path = +server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# For "sqlite3" only. cache mode setting used for connecting to the database +cache_mode = private + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +logging = false + +# How long the data proxy waits before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +timeout = 30 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +send_user_header = false + +#################################### Analytics ########################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +reporting_enabled = true + +# Set to false to disable all checks to https://grafana.com +# for new versions (grafana itself and plugins), check is used +# in some UI views to notify that grafana or plugin update exists +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get latest versions +check_for_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +google_tag_manager_id = + +#################################### Security ############################ +[security] +# disable creation of admin user on first start of grafana +disable_initial_admin_creation = false + +# default admin user, created on startup +admin_user = {{ grafana_admin_username }} + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = {{ grafana_admin_password }} + +# used for signing +secret_key = {{ grafana_signing_secret }} + +# disable gravatar profile images +disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +data_source_proxy_whitelist = + +# disable protection against brute force login attempts +disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,