DEV-873 added custom node exporter polling for EXT stage

group_vars/all/plain.yml

@@ -204,16 +204,6 @@ logstash_certificate: "{{ stage }}-elastic-stack-logstash-01"
 
 backup_directory: "/backups"
 
-blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
-blackbox_http_2xx_targets:
-- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
-- 'https://{{ stage }}-kube-awx.smardigo.digital'
-#- 'https://{{ stage }}-management-01-connect.smardigo.digital/'
-blackbox_http_2xx_additional_targets: []
-
-prometheus_federation_enabled: true
-kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
-
 get_current_date: "{{ lookup('pipe','date +%Y-%m-%d') }}"
 get_current_date_time: "{{ lookup('pipe','date +%Y-%m-%d_%H:%M') }}"
 
@@ -227,9 +217,6 @@ k8s_basic_services:
 
 selfsigned_ca_private_key_passphrase: '{{ selfsigned_ca_private_key_passphrase_vault }}'
 
-prometheus_alert_diskspaceusage_warning: 85
-prometheus_alert_pg_replication_lag: 120
-
 # hetzner upstream DNSservers
 upstream_dns_servers:
 - 185.12.64.1

group_vars/all/prometheus.yml

@@ -0,0 +1,19 @@
+---
+
+# node exporter exposes data only into the private network
+node_exporter_listen_address: "{{ stage_private_server_ip }}"
+
+# TODO the blackbox exporter shouldn't be DEV tagged at all 
+blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
+
+blackbox_http_2xx_targets:
+- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
+- 'https://{{ stage }}-kube-awx.smardigo.digital'
+#- 'https://{{ stage }}-management-01-connect.smardigo.digital/'
+blackbox_http_2xx_additional_targets: []
+
+prometheus_federation_enabled: true
+kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
+
+prometheus_alert_diskspaceusage_warning: 85
+prometheus_alert_pg_replication_lag: 120

group_vars/stage_ext/plain.yml

@@ -8,8 +8,11 @@ docker_enabled: true
 docker_config_enabled: false
 traefik_enabled: true
 filebeat_enabled: false
-node_exporter_enabled: false
+node_exporter_enabled: true
 
 shared_service_hosts: []
 
+# an ext server has no private network
+node_exporter_listen_address: "0.0.0.0"
+
 shared_service_harbor_hostname: "prodnso-harbor-01.smardigo.digital"

group_vars/stage_prodnso/prometheus.yml

@@ -7,3 +7,7 @@ prometheus_tsdb_rentention_time: '90d'
 # check firewall settings
 smardigo_connect_extra_servers:
   - ext-bdev-mpmexec-02-connect.smardigo.digital
+
+# check firewall settings
+node_exporter_extra_servers:
+  - ext-bdev-mpmexec-02.smardigo.digital

roles/node_exporter/tasks/main.yml

@@ -27,7 +27,7 @@
   lineinfile:
     path: /etc/default/prometheus-node-exporter
     regex: "^ARGS="
-    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_system }}'\""
+    line: "ARGS=\"--web.listen-address='{{ node_exporter_listen_address }}:{{ monitor_port_system }}'\""
   notify: restart node-exporter
 
 - name: "Ensure prometheus-node-exporter is running"

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -398,6 +398,31 @@ scrape_configs:
         target_label:  __address__
         replacement:   $1
 
+  - job_name: 'node-exporter-extra'
+    scheme: http
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: [
+{% for server in node_exporter_extra_servers | default([]) %}
+          '{{ server }}:{{ monitor_port_system }}',
+{% endfor %}
+        ]
+        labels:
+          env: {{ stage }}
+          project: servers
+    relabel_configs:
+      - source_labels: [job]
+        target_label:  job
+        replacement:   'node-exporter'
+      - source_labels: [__address__]
+        regex:         .*!(.*)
+        target_label:  instance
+        replacement:   $1
+      - source_labels: [__address__]
+        regex:         (.*)!.*
+        target_label:  __address__
+        replacement:   $1
+
   - job_name: 'node-exporter-external-monitoring(digitalocean)'
     scheme: http
     metrics_path: '/metrics'