DEV-269: added stuff to federate k8s-internal prometheus metrics

group_vars/all/plain.yml

@@ -31,6 +31,17 @@ traefik_enabled: true
 filebeat_enabled: true
 node_exporter_enabled: true
 
+common_apt_dependencies:
+  - mc
+  - vim
+  - zip
+  - curl
+  - htop
+  - net-tools
+  - bash-completion
+  - python3-pip
+
+common_pip_dependencies: []
 use_ssl: true
 http_s: "http{{ use_ssl | ternary('s', '', omit) }}"
 
@@ -177,3 +188,5 @@ blackbox_http_2xx_targets:
 - 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
 - 'https://{{ stage }}-management-01-connect.smardigo.digital/auth/'
 blackbox_http_2xx_additional_targets: []
+
+kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"

group_vars/prometheus/plain.yml

@@ -1,4 +1,4 @@
 ---
 
-hetzner_server_type: cpx11
+hetzner_server_type: cx21
 hetzner_server_labels: "stage={{ stage }} service=prometheus"

group_vars/stage_dev/plain.yml

@@ -131,6 +131,8 @@ shared_service_kube_node_03: "{{ stage_server_infos
   | first
   | default('-') }}"
 
+shared_service_kube_prometheus_ip: 10.0.0.20
+
 shared_service_maria_hostname: "{{ stage }}-maria-01"
 shared_service_postgres_01_hostname: "{{ stage }}-postgres-01"
 shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
@@ -149,6 +151,7 @@ shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain }}"
 shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain }}"
 shared_service_gitea_hostname: "{{ stage }}-gitea-01.{{ domain }}"
 shared_service_redis_hostname: "{{ stage }}-redis-01.{{ domain }}"
+shared_service_kube_prometheus_hostname: "{{ stage }}-kube-prometheus.{{ domain }}"
 shared_service_pdns_hostname: "{{ stage }}-pdns-01.{{ domain }}"
 shared_service_webdav_hostname: "{{ stage }}-webdav-01.{{ domain }}"
 shared_service_keycloak_hostname: "{{ stage }}-keycloak-01.{{ domain }}"
@@ -242,6 +245,10 @@ shared_service_hosts: [
   {
     ip: "{{ shared_service_pdns_ip }}",
     name: "{{ shared_service_pdns_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_prometheus_ip }}",
+    name: "{{ shared_service_kube_prometheus_hostname }}"
   }
 ]
 

group_vars/stage_qa/plain.yml

@@ -130,6 +130,7 @@ shared_service_kube_node_03: "{{ stage_server_infos
   | list
   | first
   | default('-') }}"
+shared_service_kube_prometheus_ip: 10.1.0.26
 
 shared_service_maria_hostname: "{{ stage }}-maria-01"
 shared_service_postgres_01_hostname: "{{ stage }}-postgres-01"
@@ -149,6 +150,7 @@ shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain }}"
 shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain }}"
 shared_service_gitea_hostname: "{{ stage }}-gitea-01.{{ domain }}"
 shared_service_redis_hostname: "{{ stage }}-redis-01.{{ domain }}"
+shared_service_kube_prometheus_hostname: "{{ stage }}-kube-prometheus.{{ domain }}"
 shared_service_pdns_hostname: "{{ stage }}-pdns-01.{{ domain }}"
 shared_service_webdav_hostname: "{{ stage }}-webdav-01.{{ domain }}"
 shared_service_keycloak_hostname: "{{ stage }}-keycloak-01.{{ domain }}"
@@ -242,6 +244,10 @@ shared_service_hosts: [
   {
     ip: "{{ shared_service_pdns_ip }}",
     name: "{{ shared_service_pdns_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_prometheus_ip }}",
+    name: "{{ shared_service_kube_prometheus_hostname }}"
   }
 ]
 

roles/common/tasks/main.yml

@@ -110,22 +110,24 @@
     - users
     - config
 
-- name: "Install common dependencies"
+- name: "Install common apt-dependencies"
   apt:
-    name: [
-     'mc',
-     'vim',
-     'zip',
-     'curl',
-     'htop',
-     'net-tools',
-     'bash-completion',
-    ]
+    name: "{{ item }}"
     state: 'present'
+  loop: "{{ common_apt_dependencies }}"
   when: ansible_distribution == "Ubuntu"
   tags:
     - install
 
+- name: "Install python3-pip dependencies for {{ inventory_hostname }}"
+  pip:
+    name: "{{ item }}"
+    state: present
+  become: True
+  loop: "{{ common_pip_dependencies }}"
+  tags:
+    - install
+
 - name: 'Ensures </etc/bash_completion.d> directory exists'
   file:
     state: directory

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -457,3 +457,18 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: "{{ lookup('community.general.dig', blackbox_exporter_fqdn ) }}:9115"
+##############################################
+###               federation              ####
+##############################################
+  - job_name: 'federate - kube'
+    scheme: https
+
+    honor_labels: true
+    metrics_path: '/federate'
+
+    params:
+      'match[]':
+        - '{job=~".*"}'
+
+    static_configs:
+      - targets: ['{{ kubernetes_prometheus_endpoint }}']