Removed container node-exporter, added system node-exporter, optimized aotidiscover pre-tasks.

roles/node-exporter/defaults/main.yml

@@ -1,4 +1 @@
 ---
-
-node_exporter_image_name: "prom/node-exporter"
-node_exporter_image_version: "v1.1.2"

roles/node-exporter/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 
-- name: "Send mattermost messsge"
+- name: "Send mattermost message"
   uri:
     url: "{{ mattermost_hook_smardigo }}"
     method: POST
@@ -13,52 +13,41 @@
   when:
     - send_status_messages
 
-- name: "Check if node-exporter/docker-compose.yml exists"
+- name: "Check if node-exporter/docker-compose.yml exists - TODO: remove after running setup.yml on all hosts .."
   stat:
     path: '{{ service_base_path }}/node-exporter/docker-compose.yml'
   register: check_docker_compose_file
 
-- name: "Stop node-exporter"
+- name: "Stop node-exporter - TODO: remove after running setup.yml on all hosts .."
   shell: docker-compose down
   args:
     chdir: '{{ service_base_path }}/node-exporter'
   when: check_docker_compose_file.stat.exists
   ignore_errors: yes
 
-- name: "Deploy docker templates for node-exporter"
+- name: "Remove node-exporter container fragments - TODO: remove after running setup.yml on all hosts .."
-  include_role:
+  file:
-    name: _deploy
+    state: absent
-    tasks_from: templates
+    path: '{{ service_base_path }}/node-exporter/'
-  vars:
+
-    current_config: "_docker"
+- name: Ensure prometheus-node-exporter ist installed
-    current_base_path: "{{ service_base_path }}"
+  apt:
-    current_destination: "node-exporter"
+    pkg:
-    current_owner: "{{ docker_owner }}"
+      - prometheus-node-exporter
-    current_group: "{{ docker_group }}"
+      - prometheus-node-exporter-collectors
-    current_docker: "{{ node_exporter_docker }}"
+
-
+- name: Setup prometheus-node-exporter interface bind
-- name: "Deploy service templates for node-exporter"
+  lineinfile:
-  include_role:
+    path: /etc/default/prometheus-node-exporter
-    name: _deploy
+    regex: "^ARGS="
-    tasks_from: templates
+    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_system }}'\""
-  vars:
+  register: node_exporter
-    current_config: "node-exporter"
+
-    current_base_path: "{{ service_base_path }}"
+- name: Restart prometheus if needed
-    current_destination: "node-exporter"
+  service:
-    current_owner: "{{ docker_owner }}"
+    name: prometheus-node-exporter
-    current_group: "{{ docker_group }}"
+    state: restarted
-
+  when: node_exporter.changed
-- name: "Update node-exporter"
-  shell: docker-compose pull
-  args:
-    chdir: '{{ service_base_path }}/node-exporter'
-  tags:
-    - update_deployment
-
-- name: "Start node-exporter"
-  shell: docker-compose up -d
-  args:
-    chdir: '{{ service_base_path }}/node-exporter'
 
 - name: "Send mattermost messsge"
   uri:

roles/node-exporter/vars/main.yml

@@ -1,65 +1 @@
 ---
-
-node_exporter_id: "{{ inventory_hostname }}-node-exporter"
-
-node_exporter_docker: {
-  networks: [
-    {
-      name: front-tier,
-      external: true,
-    },
-  ],
-  services: [
-    {
-      name: "{{ node_exporter_id }}",
-      image_name: "{{ node_exporter_image_name }}",
-      image_version: "{{ node_exporter_image_version }}",
-      command: [
-        '"--path.procfs=/host/proc"',
-        '"--path.sysfs=/host/sys"',
-        '"--no-collector.systemd"',
-        '"--no-collector.logind"',
-        '"--no-collector.ntp"',
-        '"--no-collector.bonding"',
-        '"--no-collector.bcache"',
-        '"--no-collector.arp"',
-        '"--no-collector.edac"',
-        '"--no-collector.infiniband"',
-        '"--no-collector.ipvs"',
-        '"--no-collector.mdadm"',
-        '"--no-collector.nfs"',
-        '"--no-collector.nfsd"',
-        '"--no-collector.wifi"',
-        '"--no-collector.hwmon"',
-        '"--no-collector.conntrack"',
-        '"--no-collector.timex"',
-        '"--no-collector.zfs"',
-        '"--collector.tcpstat"',
-        '"--collector.interrupts"',
-        '"--collector.meminfo_numa"',
-        '"--collector.processes"',
-        '"--collector.textfile"',
-        '"--collector.textfile.directory=/rootfs/textfiles"',
-        '"--collector.filesystem.ignored-mount-points"',
-        '"^/(sys|proc|dev|host|etc|run|run/lock|boot|var/lib/docker|run/docker/netns|var/lib/docker/aufs)($$|/)"',
-      ],
-      labels: [
-        '"traefik.enable=true"',
-        '"traefik.http.routers.{{ node_exporter_id }}.service={{ node_exporter_id }}"',
-        '"traefik.http.routers.{{ node_exporter_id }}.rule=Host(`{{ inventory_hostname }}.{{ domain }}`)"',
-        '"traefik.http.routers.{{ node_exporter_id }}.entrypoints=monitoring-system"',
-        '"traefik.http.routers.{{ node_exporter_id }}.tls=true"',
-        '"traefik.http.routers.{{ node_exporter_id }}.tls.certresolver=letsencrypt"',
-        '"traefik.http.services.{{ node_exporter_id }}.loadbalancer.server.port={{ service_port_node_exporter }}"',
-      ],
-      volumes: [
-        '"/proc:/host/proc:ro"',
-        '"/sys:/host/sys:ro"',
-        '"/:/rootfs:ro"',
-      ],
-      networks: [
-        '"front-tier"'
-      ]
-    }
-  ]
-}

roles/traefik/vars/main.yml

@@ -56,10 +56,6 @@ traefik_docker: {
           external: "0.0.0.0:{{ monitor_port_service }}",
           internal: "{{ monitor_port_service }}"
         },
-        {
-          external: "0.0.0.0:{{ monitor_port_system }}",
-          internal: "{{ monitor_port_system }}"
-        },
         {
           external: "0.0.0.0:{{ monitor_port_docker }}",
           internal: "{{ monitor_port_docker }}"

tasks/autodiscover_pre_tasks.yml

@@ -22,24 +22,6 @@
   when:
     - debug
 
-- name: "Reading hetzner server infos for stage <{{ stage }}>"
-  set_fact:
-    stage_server_infos: "{{ hetzner_servers.json.servers | json_query(querystr) | default([]) }}"
-  vars:
-    querystr: "[*].{ name: name, public_ip: public_net.ipv4.ip, private_ip: private_net[0].ip }"
-  delegate_to: 127.0.0.1
-  become: false
-  tags:
-    - autodiscover
-
-- name: "Reading hetzner server infos for stage <{{ stage }}>"
-  debug:
-    msg: "{{ stage_server_infos }}"
-  delegate_to: 127.0.0.1
-  become: false
-  tags:
-    - autodiscover
-
 - name: "Reading hetzner private network infos for stage <{{ stage }}>"
   uri:
     url: "https://api.hetzner.cloud/v1/networks?label_selector=stage={{ stage }}"
@@ -82,14 +64,17 @@
   when:
     - debug
 
-# TODO always match first private_net network is wrong
-#   use stage_private_network_id to match the right network
-#   expression [?network=`{{ stage_private_network_id }}`] does not work
 - name: "Reading private ip address for {{ inventory_hostname }}"
   set_fact:
-    stage_private_server_ip: "{{ hetzner_servers.json.servers | json_query(querystr) | first }}"
+    stage_private_server_ip: "{% for server in hetzner_servers.json.servers %}\
-  vars:
+      {% if server.name == inventory_hostname %}\
-    querystr: "[?name=='{{ inventory_hostname }}'].private_net[0].ip"
+        {% for private_net in server.private_net %}\
+          {% if private_net.network == stage_private_network_id|int %}\
+            {{ private_net.ip }}\
+          {% endif %}\
+          {% endfor %}\
+        {% endif %}\
+      {% endfor %}"
   delegate_to: 127.0.0.1
   become: false
   tags:
@@ -114,3 +99,32 @@
     - autodiscover
   when:
     - debug
+
+- name: "Reading hetzner server infos for stage <{{ stage }}>"
+  set_fact:
+    stage_server_infos: "\
+      {% set  list= [] %}\
+      {% for server in hetzner_servers.json.servers %}\
+        {% for private_net in server.private_net %}\
+          {% if private_net.network == stage_private_network_id|int %}\
+            {{ list.append({'service': server.labels.service, 'private_ip': private_net.ip, 'public_ip': server.public_net.ipv4.ip, 'name': server.name}) }}\
+          {% endif %}\
+        {% endfor %}\
+      {% endfor %}\
+      {{ list|list }}"
+  delegate_to: 127.0.0.1
+  become: false
+  tags:
+    - autodiscover
+  when:
+    - debug
+
+- name: "Reading hetzner server infos for stage <{{ stage }}>"
+  debug:
+    msg: "{{ stage_server_infos }}"
+  delegate_to: 127.0.0.1
+  become: false
+  tags:
+    - autodiscover
+  when:
+    - debug

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -223,7 +223,7 @@ scrape_configs:
 ##############################################
 
   - job_name: 'node-exporter'
-    scheme: {{ http_s }}
+    scheme: http
     metrics_path: '/metrics'
     static_configs:
       - targets: [
@@ -233,7 +233,7 @@ scrape_configs:
   and hostvars[host].node_exporter_enabled)
   or (hostvars[host].node_exporter_enabled is not defined)
 %}
-          '{{ host }}.{{ domain }}:{{ monitor_port_system }}',
+          '{{ (stage_server_infos | selectattr("name", "equalto", host) | first).private_ip }}:{{ monitor_port_system }}!{{ host }}.{{ domain }}',
 {% endif %}
 {% endfor %}
         ]
@@ -245,9 +245,13 @@ scrape_configs:
         target_label:  job
         replacement:   'node-exporter'
       - source_labels: [__address__]
-        regex:         (.*):.*
+        regex:         .*!(.*)
         target_label:  instance
         replacement:   $1
+      - source_labels: [__address__]
+        regex:         (.*)!.*
+        target_label:  __address__
+        replacement:   $1
 
 ##############################################
 ###               Databases               ####