feat: added elasticsearch exporter setup

group_vars/stage_dev/plain.yml

@@ -2,9 +2,12 @@
 
 stage: "dev"
 
+hetzner_server_labels: "stage={{ stage }}"
+
 alertmanager_channel_smardigo: "#monitoring-qa"
 
 # TODO read configuration with hetzner rest api
+logstash_hostname: "logstash-dev-elastic-stack-01"
 filebeat_extra_hosts: [
   {
     hostname: logstash-dev-elastic-stack-01,
@@ -19,3 +22,28 @@ filebeat_extra_hosts: [
     ip: 10.0.0.4,
   },
 ]
+
+# TODO read configuration with hetzner rest api
+keycloak_hostname: "dev-keycloak-01.smardigo.digital"
+connect_extra_hosts: [
+  {
+    hostname: "{{ keycloak_hostname }}",
+    ip: 10.1.0.2,
+  }
+]
+
+# TODO read configuration with hetzner rest api
+elasticsearch_extra_hosts: [
+  {
+    hostname: es-dev-elastic-stack-01,
+    ip: 10.0.0.2,
+  },
+  {
+    hostname: es-dev-elastic-stack-02,
+    ip: 10.0.0.3
+  },
+  {
+    hostname: es-dev-elastic-stack-03,
+    ip: 10.0.0.4,
+  },
+]

host_vars/dev-connect-01.yml

@@ -1,7 +1,15 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=connect"
+
 connect_auth_module: oidc
 connect_oidc_client_id: connect-01
 connect_oidc_client_secret: 9e234965-1041-4653-8a0e-db964c04bc26
 connect_oidc_registration_id: connect-01
-connect_oidc_issuer_uri: https://dev-keycloak-01.smardigo.digital/auth/realms/smardigo-01
+connect_oidc_issuer_uri: https://{{ keycloak_hostname }}/auth/realms/smardigo-01
+
+connect_password_change_url: https://{{ keycloak_hostname }}/auth/realms/smardigo-01/account/password
+connect_iam_user_management_url: https://{{ keycloak_hostname }}/auth/admin/smardigo-01/console
+
+spring_profiles_include_suffix: ",hetzner"
+ribbon_display_on_active_profiles: "hetzner"

host_vars/dev-connect-02.yml

@@ -1,7 +1,15 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=connect"
+
 connect_auth_module: oidc
 connect_oidc_client_id: connect-02
 connect_oidc_client_secret: 9e234965-1041-4653-8a0e-db964c04bc26
 connect_oidc_registration_id: connect-02
-connect_oidc_issuer_uri: https://dev-keycloak-01.smardigo.digital/auth/realms/smardigo-01
+connect_oidc_issuer_uri: https://{{ keycloak_hostname }}/auth/realms/smardigo-01
+
+connect_password_change_url: https://{{ keycloak_hostname }}/auth/realms/smardigo-01/account/password
+connect_iam_user_management_url: https://{{ keycloak_hostname }}/auth/admin/smardigo-01/console
+
+spring_profiles_include_suffix: ",hetzner"
+ribbon_display_on_active_profiles: "hetzner"

host_vars/dev-connect-03.yml

@@ -1,7 +1,15 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=connect"
+
 connect_auth_module: oidc
 connect_oidc_client_id: connect-03
 connect_oidc_client_secret: 9e234965-1041-4653-8a0e-db964c04bc26
 connect_oidc_registration_id: connect-03
-connect_oidc_issuer_uri: https://dev-keycloak-01.smardigo.digital/auth/realms/smardigo-01
+connect_oidc_issuer_uri: https://{{ keycloak_hostname }}/auth/realms/smardigo-01
+
+connect_password_change_url: https://{{ keycloak_hostname }}/auth/realms/smardigo-01/account/password
+connect_iam_user_management_url: https://{{ keycloak_hostname }}/auth/admin/smardigo-01/console
+
+spring_profiles_include_suffix: ",hetzner"
+ribbon_display_on_active_profiles: "hetzner"

host_vars/dev-connect-04.yml

@@ -1,7 +1,15 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=connect"
+
 connect_auth_module: oidc
 connect_oidc_client_id: connect-04
 connect_oidc_client_secret: 9e234965-1041-4653-8a0e-db964c04bc26
 connect_oidc_registration_id: connect-04
-connect_oidc_issuer_uri: https://dev-keycloak-01.smardigo.digital/auth/realms/smardigo-01
+connect_oidc_issuer_uri: https://{{ keycloak_hostname }}/auth/realms/smardigo-01
+
+connect_password_change_url: https://{{ keycloak_hostname }}/auth/realms/smardigo-01/account/password
+connect_iam_user_management_url: https://{{ keycloak_hostname }}/auth/admin/smardigo-01/console
+
+spring_profiles_include_suffix: ",hetzner"
+ribbon_display_on_active_profiles: "hetzner"

host_vars/dev-elastic-stack-01.yml

@@ -1,5 +1,7 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=elastic"
+
 hetzner_server_type: cx31
 
 hetzner_ssh_keys:

host_vars/dev-elastic-stack-02.yml

@@ -1,5 +1,7 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=elastic"
+
 hetzner_server_type: cx31
 
 hetzner_ssh_keys:

host_vars/dev-elastic-stack-03.yml

@@ -1,5 +1,7 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=elastic"
+
 hetzner_server_type: cx31
 
 hetzner_ssh_keys:

host_vars/dev-keycloak-01.yml

@@ -1,5 +1,7 @@
 ---
 
+hetzner_server_labels: "stage={{ stage }} service=keycloak"
+
 keycloak: {
   realms: [
     {

networks.yml

@@ -0,0 +1,21 @@
+---
+
+- name: 'apply setup to {{ host | default("all") }}'
+  hosts: '{{ host | default("all") }}'
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: no
+  become: no
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+
+  roles:
+    - role: hetzner-network
+      vars:
+        hetzner_state: 'started'
+      when: "'hcloud' in group_names"

provisioning.yml

@@ -14,6 +14,47 @@
           - ansible_version.minor >= 10
         msg: "The ansible version has to be at least ({{ ansible_version.full }})"
 
+    - name: Get all Firewalls from Hetzner
+      uri:
+        url: "https://api.hetzner.cloud/v1/firewalls"
+        headers:
+          accept: application/json
+          authorization: Bearer {{ hetzner_authentication_token }}
+        return_content: yes
+      register: hetzner_firewalls_response
+      delegate_to: 127.0.0.1
+      run_once: true
+
+    - name: "Gather current server infos"
+      hcloud_server_info:
+        api_token: "{{ hetzner_authentication_token }}"
+      register: hetzner_server_infos
+      delegate_to: 127.0.0.1
+      become: false
+
+    - name: "Set current server infos as fact: hetzner_server_infos_json"
+      set_fact: 
+        hetzner_server_infos_json: "{{ hetzner_server_infos.hcloud_server_info }}"
+      delegate_to: 127.0.0.1
+      become: false
+    
+    - name: Save firewall entries as variable (fact)
+      set_fact: 
+        hetzner_firewalls_response_json: "{{ hetzner_firewalls_response.json }}"
+      run_once: true
+    
+    - name: Parse firewall entries
+      set_fact:  
+        firewall_records: "{{ hetzner_firewalls_response_json.firewalls | json_query(jmesquery) }}"
+      vars:
+        jmesquery: '[*].{id: id, name: name}'
+      run_once: true
+    
+    - name: Print firewall entries
+      debug:
+        msg: "{{ firewall_records }}"
+      run_once: true
+
   roles:
     - role: hcloud
       when: "'hcloud' in group_names"

roles/common/tasks/main.yml

@@ -113,12 +113,29 @@
      'curl',
      'htop',
      'net-tools',
+     'bash-completion',
     ]
     state: 'present'
   when: ansible_distribution == "Ubuntu"
   tags:
     - install
 
+- name: Download docker-compose bash completion
+  get_url:
+    url: https://raw.githubusercontent.com/docker/cli/v20.10.6/contrib/completion/bash/docker
+    dest: /usr/share/bash-completion/docker
+    mode: '644'
+  tags:
+    - install
+
+- name: Download docker-compose bash completion
+  get_url:
+    url: https://raw.githubusercontent.com/docker/compose/1.29.1/contrib/completion/bash/docker-compose
+    dest: /usr/share/bash-completion/docker-compose
+    mode: '644'
+  tags:
+    - install
+
 - name: "Upgrade all packages"
   apt:
     name: '*'

roles/elasticsearch-exporter/defaults/main.yml

@@ -0,0 +1,47 @@
+---
+
+elasticsearch_exporter_id: "{{ service_name }}-elasticsearch-exporter"
+
+elasticsearch_exporter_image_name: "justwatch/elasticsearch_exporter"
+elasticsearch_exporter_image_version: "latest"
+
+elasticsearch_exporter_docker: {
+  networks: [
+    {
+      name: back-tier,
+      external: true,
+    },
+    {
+      name: front-tier,
+      external: true,
+    },
+  ],
+  services: [
+    {
+      name: "{{ elasticsearch_exporter_id }}",
+      image_name: "{{ elasticsearch_exporter_image_name }}",
+      image_version: "{{ elasticsearch_exporter_image_version }}",
+      command: [
+        '"--es.ca=/certificates/ca.crt"',
+        '"--es.uri=https://logstash-ingest:tH1iSiSas3cREt.Passw0rt@es-dev-elastic-stack-01:9200"',
+      ],
+      labels: [
+        '"traefik.enable=true"',
+        '"traefik.http.routers.{{ elasticsearch_exporter_id }}.service={{ elasticsearch_exporter_id }}"',
+        '"traefik.http.routers.{{ elasticsearch_exporter_id }}.rule=Host(`{{ service_name }}.{{ domain }}`)"',
+        '"traefik.http.routers.{{ elasticsearch_exporter_id }}.entrypoints=admin-docker"',
+        '"traefik.http.routers.{{ elasticsearch_exporter_id }}.tls=true"',
+        '"traefik.http.routers.{{ elasticsearch_exporter_id }}.tls.certresolver=letsencrypt"',
+        '"traefik.http.services.{{ elasticsearch_exporter_id }}.loadbalancer.server.port=9114"',
+      ],
+      volumes: [
+        '"./certs:/certificates:ro"',
+      ],
+      networks: [
+        '"back-tier"',
+        '"front-tier"',
+      ],
+      extra_hosts: "{{ elasticsearch_extra_hosts | default([]) }}",
+    }
+  ]
+}

roles/elasticsearch-exporter/handlers/main.yml

@@ -0,0 +1 @@
+---

roles/elasticsearch-exporter/meta/main.yml

@@ -0,0 +1 @@
+---

roles/elasticsearch-exporter/tasks/main.yml

@@ -0,0 +1,63 @@
+---
+
+- name: "Send mattermost messsge"
+  uri:
+    url: "{{ mattermost_hook_smardigo }}"
+    method: POST
+    body: "{{ lookup('template','mattermost-deploy-start.json.j2') }}"
+    body_format: json
+    headers:
+      Content-Type: "application/json"
+  delegate_to: 127.0.0.1
+  become: false
+  when:
+    - send_status_messages
+
+- name: "Check if {{ role_name }}/docker-compose.yml exists"
+  stat:
+    path: '{{ service_base_path }}/{{ role_name }}/docker-compose.yml'
+  register: check_docker_compose_file
+
+- name: "Stop {{ role_name }}"
+  shell: docker-compose down
+  args:
+    chdir: '{{ service_base_path }}/{{ role_name }}'
+  when: check_docker_compose_file.stat.exists
+  ignore_errors: yes
+
+- name: "Deploy service configuration for {{ role_name }}"
+  include_role:
+    name: _deploy
+    tasks_from: configs
+  vars:
+    current_config: "elasticsearch-exporter"
+    current_base_path: "{{ service_base_path }}"
+    current_destination: "elasticsearch-exporter"
+    current_owner: "{{ docker_owner }}"
+    current_group: "{{ docker_group }}"
+    current_docker: "{{ elasticsearch_exporter_docker }}"
+
+- name: "Update {{ role_name }}"
+  shell: docker-compose pull
+  args:
+    chdir: '{{ service_base_path }}/{{ role_name }}'
+  tags:
+    - update_deployment
+
+- name: "Start {{ role_name }}"
+  shell: docker-compose up -d
+  args:
+    chdir: '{{ service_base_path }}/{{ role_name }}'
+
+- name: "Send mattermost messsge"
+  uri:
+    url: "{{ mattermost_hook_smardigo }}"
+    method: POST
+    body: "{{ lookup('template','mattermost-deploy-end.json.j2') }}"
+    body_format: json
+    headers:
+      Content-Type: "application/json"
+  delegate_to: 127.0.0.1
+  become: false
+  when:
+    - send_status_messages

roles/elasticsearch-exporter/vars/main.yml

@@ -0,0 +1 @@
+---

roles/filebeat/defaults/main.yaml

@@ -1,7 +1,5 @@
 ---
 
-logstash_hostname: "logstash-dev-elastic-stack-01"
-
 filebeat_image_name: "docker.elastic.co/beats/filebeat"
 filebeat_image_version: "7.12.0"
 

roles/hcloud/tasks/configure-firewall.yml

@@ -2,34 +2,6 @@
 
 ### tags:
 
-- name: Get all Firewalls from Hetzner
-  uri:
-    url: "https://api.hetzner.cloud/v1/firewalls"
-    headers:
-      accept: application/json
-      authorization: Bearer {{ hetzner_authentication_token }}
-    return_content: yes
-  register: hetzner_firewalls_response
-  delegate_to: 127.0.0.1
-  run_once: true
-
-- name: Save firewall entries as variable (fact)
-  set_fact: 
-    hetzner_firewalls_response_json: "{{ hetzner_firewalls_response.json }}"
-  run_once: true
-
-- name: Parse firewall entries
-  set_fact:  
-    firewall_records: "{{ hetzner_firewalls_response_json.firewalls | json_query(jmesquery) }}"
-  vars:
-    jmesquery: '[*].{id: id, name: name}'
-  run_once: true
-
-- name: Print firewall entries
-  debug:
-    msg: "{{ firewall_records }}"
-  run_once: true
-
 - name: Read firewall entry for {{ current_firewall_name }}
   set_fact:  
     firewall_record: "{{ firewall_records | selectattr('name', 'equalto', current_firewall_name) | list | first | default({'name': '-', 'id': '-'}) }}"

roles/hcloud/tasks/main.yml

@@ -2,17 +2,6 @@
 
 ### tags:
 
-- name: "Setup firewall"
-  include_tasks: configure-firewall.yml
-  vars:
-    current_firewall_name: '{{ current_firewall }}'
-  with_items:
-    - 'default'
-    - 'kibana'
-    - 'monitoring'
-  loop_control:
-    loop_var: current_firewall
-
 - name: Create new server {{ inventory_hostname }}
   hetzner.hcloud.hcloud_server:
     api_token: "{{ hetzner_authentication_token }}"
@@ -20,10 +9,22 @@
     server_type: "{{ hetzner_server_type }}"
     image: "{{ hetzner_server_image }}"
     ssh_keys: "{{ hetzner_ssh_keys }}"
+    labels: "{{ hetzner_server_labels }}"
     location: nbg1
     state: present
   delegate_to: 127.0.0.1
 
+- name: "Setup firewall"
+  include_tasks: configure-firewall.yml
+  vars:
+    current_firewall_name: '{{ current_firewall }}'
+  with_items:
+    - 'default'
+    - 'kibana'
+    - 'monitoring'
+  loop_control:
+    loop_var: current_firewall
+
 - name: Read ip for {{ inventory_hostname }}
   set_fact:
     stage_server_ip: "{{ item.ipv4_address }}"

roles/hetzner-network/defaults/main.yml

@@ -0,0 +1 @@
+---

roles/hetzner-network/handlers/main.yml

@@ -0,0 +1 @@
+---

roles/hetzner-network/meta/main.yml

@@ -0,0 +1 @@
+---

roles/hetzner-network/tasks/main.yml

@@ -0,0 +1,61 @@
+---
+
+### tags:
+
+# TODO remove static ip configuration
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-elastic-stack
+    server: dev-elastic-stack-01
+    ip: 10.0.0.2
+    state: present
+  delegate_to: 127.0.0.1
+
+# TODO remove static ip configuration
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-elastic-stack
+    server: dev-elastic-stack-02
+    ip: 10.0.0.3
+    state: present
+  delegate_to: 127.0.0.1
+
+# TODO remove static ip configuration
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-elastic-stack
+    server: dev-elastic-stack-03
+    ip: 10.0.0.4
+    state: present
+  delegate_to: 127.0.0.1
+
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-elastic-stack
+    server: "{{ item }}"
+    state: present
+  with_items: "{{ groups['hcloud'] | difference(groups['elastic']) }}"
+  delegate_to: 127.0.0.1
+
+# TODO remove static ip configuration
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-keycloak
+    server: dev-keycloak-01
+    ip: 10.1.0.2
+    state: present
+  delegate_to: 127.0.0.1
+
+- name: Create a server network and specify the ip address
+  hcloud_server_network:
+    api_token: "{{ hetzner_authentication_token }}"
+    network: dev-keycloak
+    server: "{{ item }}"
+    state: present
+  with_items: "{{ groups['connect'] }}"
+  delegate_to: 127.0.0.1

roles/hetzner-network/vars/main.yml

@@ -0,0 +1 @@
+---

roles/keycloak/defaults/main.yml

@@ -81,7 +81,7 @@ keycloak_docker: {
           internal: "{{ service_port_keycloak }}",
         },
       ],
-      extra_hosts: "{{ connect_extra_hosts | default([]) }}",
+      extra_hosts: "{{ keycloak_extra_hosts | default([]) }}",
     },
     {
       name: "{{ keycloak_postgres_id }}",

roles/prometheus/defaults/main.yml

@@ -59,7 +59,7 @@ prometheus_docker: {
         '"--web.console.templates=/usr/share/prometheus/consoles"',
         '"--web.external-url={{ http_s}}://{{ service_name }}-prometheus.{{ domain }}"',
         '"--web.enable-lifecycle"',
-        '"--storage.tsdb.retention.time=30w"',
+        '"--storage.tsdb.retention.time=15w"',
       ],
       volumes: [
         '"./config/prometheus/:/etc/prometheus/:ro"',

setup.yml

@@ -63,7 +63,7 @@
   roles:
     - role: ansible-role-docker
       vars:
-        docker_compose_version: '1.25.5'
+        docker_compose_version: '1.29.1'
         docker_compose_path: '/usr/bin/docker-compose'
         docker_users: '{{ smardigo_plattform_users }}'
 
@@ -81,6 +81,11 @@
       tags:
         - node-exporter
 
+    - role: elasticsearch-exporter
+      when: "'elastic' in group_names"
+      tags:
+        - elasticsearch-exporter
+
     - role: traefik
       when: traefik_enabled | default(True)
       tags:

templates/elasticsearch-exporter/certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSjCCAjKgAwIBAgIVAO1gvUalebylIyFuIAZC6bfhz04QMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTIxMDQxODExMDkwOFoXDTIyMDQxODExMDkwOFowNDEyMDAG
+A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCLcbwtcUwHBNBOlLoZA+lH
+xMoOrrySQNRRyLw/hV+KpW1YncCgVq3dGEOjOC3lS1B55+sZfjEn7EKfDtrZN6Pf
+0Ot22/GV3r+fJi72njBfay1Cep8OCJxNOx9i0N3XO2GN6IYPMEpkqFj8nySpAgh3
+70hILu3QMov2I2rWXMzE3yV6Pi7OQ151Fa8vZ1HTXkpjO7Rxyt36cXLB7slj6Uxo
+72cO0WphRV6e24Fx5iRLlAs7WdXDOSUXZfIFBiZGYvuZIgbAw9M9ZR5536eXBFuQ
+MuwLiP5g+D5GZbal5enRUShBknRP9Xvnxv7OOnPhMXVHMTsM9feqxVzmhRPp4XBz
+AgMBAAGjUzBRMB0GA1UdDgQWBBRJ5gyop7tp96EV6O/FHIY2P3T7pzAfBgNVHSME
+GDAWgBRJ5gyop7tp96EV6O/FHIY2P3T7pzAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQBEgehvsAW5r1/nogmIhhRVl5rZcy9mnbxsy/9udU1zBTEe
+ZhgCCqOx6xffXUWSvVXw3BUUizCvB5nSHCYBt3H2f8sdPXO54b5mcld/2n/D39yw
+HSODGmgkbEVjXK1Qx4xYDRHJnOuyExWQ1D7Y7HocgtIRySFdG/h7en5SM2ooJ7fa
+pPtCp8f1tHHuKCjKhgC/+wlvEZFHOWcu6Hyh1FtWHwD3uu9Tj3VRKMvW0u+KQ4mC
+aNEuHUEKzgwXRZvBG8Y5k35bFf9EVulTsD2fOTMWrD9CEdctQIfQnn1Oy3s43x39
+94DgEx78H/5fGkUDjqljXp1RBDeNJV7+tssRMISL
+-----END CERTIFICATE-----

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -108,3 +108,28 @@ scrape_configs:
         regex:         (.*):.*
         target_label:  instance
         replacement:   $1
+
+##############################################
+###               Databases               ####
+##############################################
+
+  - job_name: 'elasticsearch-exporter'
+    scheme: {{ http_s }}
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: [
+{% for host in groups['elastic'] | default([]) %}
+          '{{ host }}.{{ domain }}:9083',
+{% endfor %}
+        ]
+        labels:
+          env: {{ stage }}
+          project: smardigo
+    relabel_configs:
+      - source_labels: [job]
+        target_label:  job
+        replacement:   'elasticsearch-exporter'
+      - source_labels: [__address__]
+        regex:         (.*):.*
+        target_label:  instance
+        replacement:   $1