From e4a391be7f072c94f3a35528bdbbba42196ca7f1 Mon Sep 17 00:00:00 2001
From: "Ketelsen, Sven" <sven.ketelsen@netgo.de>
Date: Tue, 14 Feb 2023 08:49:31 +0000
Subject: [PATCH] DEV-873 added custom node exporter polling for EXT stage

---
 group_vars/all/plain.yml                      | 13 ----------
 group_vars/all/prometheus.yml                 | 19 ++++++++++++++
 group_vars/stage_ext/plain.yml                |  5 +++-
 group_vars/stage_prodnso/prometheus.yml       |  4 +++
 roles/node_exporter/tasks/main.yml            |  2 +-
 .../config/prometheus/prometheus.yml.j2       | 25 +++++++++++++++++++
 6 files changed, 53 insertions(+), 15 deletions(-)
 create mode 100644 group_vars/all/prometheus.yml

diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml
index e5ce920..e53c197 100644
--- a/group_vars/all/plain.yml
+++ b/group_vars/all/plain.yml
@@ -204,16 +204,6 @@ logstash_certificate: "{{ stage }}-elastic-stack-logstash-01"
 
 backup_directory: "/backups"
 
-blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
-blackbox_http_2xx_targets:
-- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
-- 'https://{{ stage }}-kube-awx.smardigo.digital'
-#- 'https://{{ stage }}-management-01-connect.smardigo.digital/'
-blackbox_http_2xx_additional_targets: []
-
-prometheus_federation_enabled: true
-kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
-
 get_current_date: "{{ lookup('pipe','date +%Y-%m-%d') }}"
 get_current_date_time: "{{ lookup('pipe','date +%Y-%m-%d_%H:%M') }}"
 
@@ -226,9 +216,6 @@ k8s_basic_services:
   - containerd
 
 selfsigned_ca_private_key_passphrase: '{{ selfsigned_ca_private_key_passphrase_vault }}'
-  
-prometheus_alert_diskspaceusage_warning: 85
-prometheus_alert_pg_replication_lag: 120
 
 # hetzner upstream DNSservers
 upstream_dns_servers:
diff --git a/group_vars/all/prometheus.yml b/group_vars/all/prometheus.yml
new file mode 100644
index 0000000..8737312
--- /dev/null
+++ b/group_vars/all/prometheus.yml
@@ -0,0 +1,19 @@
+---
+
+# node exporter exposes data only into the private network
+node_exporter_listen_address: "{{ stage_private_server_ip }}"
+
+# TODO the blackbox exporter shouldn't be DEV tagged at all 
+blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
+
+blackbox_http_2xx_targets:
+- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
+- 'https://{{ stage }}-kube-awx.smardigo.digital'
+#- 'https://{{ stage }}-management-01-connect.smardigo.digital/'
+blackbox_http_2xx_additional_targets: []
+
+prometheus_federation_enabled: true
+kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
+
+prometheus_alert_diskspaceusage_warning: 85
+prometheus_alert_pg_replication_lag: 120
diff --git a/group_vars/stage_ext/plain.yml b/group_vars/stage_ext/plain.yml
index fd6d26a..2d32474 100644
--- a/group_vars/stage_ext/plain.yml
+++ b/group_vars/stage_ext/plain.yml
@@ -8,8 +8,11 @@ docker_enabled: true
 docker_config_enabled: false
 traefik_enabled: true
 filebeat_enabled: false
-node_exporter_enabled: false
+node_exporter_enabled: true
 
 shared_service_hosts: []
 
+# an ext server has no private network
+node_exporter_listen_address: "0.0.0.0"
+
 shared_service_harbor_hostname: "prodnso-harbor-01.smardigo.digital"
diff --git a/group_vars/stage_prodnso/prometheus.yml b/group_vars/stage_prodnso/prometheus.yml
index 124e3be..8965b6c 100644
--- a/group_vars/stage_prodnso/prometheus.yml
+++ b/group_vars/stage_prodnso/prometheus.yml
@@ -7,3 +7,7 @@ prometheus_tsdb_rentention_time: '90d'
 # check firewall settings
 smardigo_connect_extra_servers:
   - ext-bdev-mpmexec-02-connect.smardigo.digital
+
+# check firewall settings
+node_exporter_extra_servers:
+  - ext-bdev-mpmexec-02.smardigo.digital
diff --git a/roles/node_exporter/tasks/main.yml b/roles/node_exporter/tasks/main.yml
index f4e70fa..c5e7d3c 100644
--- a/roles/node_exporter/tasks/main.yml
+++ b/roles/node_exporter/tasks/main.yml
@@ -27,7 +27,7 @@
   lineinfile:
     path: /etc/default/prometheus-node-exporter
     regex: "^ARGS="
-    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_system }}'\""
+    line: "ARGS=\"--web.listen-address='{{ node_exporter_listen_address }}:{{ monitor_port_system }}'\""
   notify: restart node-exporter
 
 - name: "Ensure prometheus-node-exporter is running"
diff --git a/templates/prometheus/config/prometheus/prometheus.yml.j2 b/templates/prometheus/config/prometheus/prometheus.yml.j2
index ce1dab5..c560b75 100644
--- a/templates/prometheus/config/prometheus/prometheus.yml.j2
+++ b/templates/prometheus/config/prometheus/prometheus.yml.j2
@@ -398,6 +398,31 @@ scrape_configs:
         target_label:  __address__
         replacement:   $1
 
+  - job_name: 'node-exporter-extra'
+    scheme: http
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: [
+{% for server in node_exporter_extra_servers | default([]) %}
+          '{{ server }}:{{ monitor_port_system }}',
+{% endfor %}
+        ]
+        labels:
+          env: {{ stage }}
+          project: servers
+    relabel_configs:
+      - source_labels: [job]
+        target_label:  job
+        replacement:   'node-exporter'
+      - source_labels: [__address__]
+        regex:         .*!(.*)
+        target_label:  instance
+        replacement:   $1
+      - source_labels: [__address__]
+        regex:         (.*)!.*
+        target_label:  __address__
+        replacement:   $1
+
   - job_name: 'node-exporter-external-monitoring(digitalocean)'
     scheme: http
     metrics_path: '/metrics'