From 1bfcac5646d159867ac1ac715a26bef87455ffeb Mon Sep 17 00:00:00 2001 From: Peter Heise Date: Fri, 3 Sep 2021 15:38:35 +0200 Subject: [PATCH] Removed container node-exporter, added system node-exporter, optimized aotidiscover pre-tasks. --- roles/node-exporter/defaults/main.yml | 3 - roles/node-exporter/tasks/main.yml | 63 ++++++++---------- roles/node-exporter/vars/main.yml | 64 ------------------- roles/prometheus_config/tasks/main.yml | 0 roles/traefik/vars/main.yml | 6 +- tasks/autodiscover_pre_tasks.yml | 64 +++++++++++-------- .../config/prometheus/prometheus.yml.j2 | 10 ++- 7 files changed, 73 insertions(+), 137 deletions(-) create mode 100644 roles/prometheus_config/tasks/main.yml diff --git a/roles/node-exporter/defaults/main.yml b/roles/node-exporter/defaults/main.yml index b1d819e..ed97d53 100644 --- a/roles/node-exporter/defaults/main.yml +++ b/roles/node-exporter/defaults/main.yml @@ -1,4 +1 @@ --- - -node_exporter_image_name: "prom/node-exporter" -node_exporter_image_version: "v1.1.2" diff --git a/roles/node-exporter/tasks/main.yml b/roles/node-exporter/tasks/main.yml index 3fd0023..402221b 100644 --- a/roles/node-exporter/tasks/main.yml +++ b/roles/node-exporter/tasks/main.yml @@ -1,6 +1,6 @@ --- -- name: "Send mattermost messsge" +- name: "Send mattermost message" uri: url: "{{ mattermost_hook_smardigo }}" method: POST @@ -13,52 +13,41 @@ when: - send_status_messages -- name: "Check if node-exporter/docker-compose.yml exists" +- name: "Check if node-exporter/docker-compose.yml exists - TODO: remove after running setup.yml on all hosts .." stat: path: '{{ service_base_path }}/node-exporter/docker-compose.yml' register: check_docker_compose_file -- name: "Stop node-exporter" +- name: "Stop node-exporter - TODO: remove after running setup.yml on all hosts .." shell: docker-compose down args: chdir: '{{ service_base_path }}/node-exporter' when: check_docker_compose_file.stat.exists ignore_errors: yes -- name: "Deploy docker templates for node-exporter" - include_role: - name: _deploy - tasks_from: templates - vars: - current_config: "_docker" - current_base_path: "{{ service_base_path }}" - current_destination: "node-exporter" - current_owner: "{{ docker_owner }}" - current_group: "{{ docker_group }}" - current_docker: "{{ node_exporter_docker }}" - -- name: "Deploy service templates for node-exporter" - include_role: - name: _deploy - tasks_from: templates - vars: - current_config: "node-exporter" - current_base_path: "{{ service_base_path }}" - current_destination: "node-exporter" - current_owner: "{{ docker_owner }}" - current_group: "{{ docker_group }}" - -- name: "Update node-exporter" - shell: docker-compose pull - args: - chdir: '{{ service_base_path }}/node-exporter' - tags: - - update_deployment - -- name: "Start node-exporter" - shell: docker-compose up -d - args: - chdir: '{{ service_base_path }}/node-exporter' +- name: "Remove node-exporter container fragments - TODO: remove after running setup.yml on all hosts .." + file: + state: absent + path: '{{ service_base_path }}/node-exporter/' + +- name: Ensure prometheus-node-exporter ist installed + apt: + pkg: + - prometheus-node-exporter + - prometheus-node-exporter-collectors + +- name: Setup prometheus-node-exporter interface bind + lineinfile: + path: /etc/default/prometheus-node-exporter + regex: "^ARGS=" + line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_system }}'\"" + register: node_exporter + +- name: Restart prometheus if needed + service: + name: prometheus-node-exporter + state: restarted + when: node_exporter.changed - name: "Send mattermost messsge" uri: diff --git a/roles/node-exporter/vars/main.yml b/roles/node-exporter/vars/main.yml index 1868fd4..ed97d53 100644 --- a/roles/node-exporter/vars/main.yml +++ b/roles/node-exporter/vars/main.yml @@ -1,65 +1 @@ --- - -node_exporter_id: "{{ inventory_hostname }}-node-exporter" - -node_exporter_docker: { - networks: [ - { - name: front-tier, - external: true, - }, - ], - services: [ - { - name: "{{ node_exporter_id }}", - image_name: "{{ node_exporter_image_name }}", - image_version: "{{ node_exporter_image_version }}", - command: [ - '"--path.procfs=/host/proc"', - '"--path.sysfs=/host/sys"', - '"--no-collector.systemd"', - '"--no-collector.logind"', - '"--no-collector.ntp"', - '"--no-collector.bonding"', - '"--no-collector.bcache"', - '"--no-collector.arp"', - '"--no-collector.edac"', - '"--no-collector.infiniband"', - '"--no-collector.ipvs"', - '"--no-collector.mdadm"', - '"--no-collector.nfs"', - '"--no-collector.nfsd"', - '"--no-collector.wifi"', - '"--no-collector.hwmon"', - '"--no-collector.conntrack"', - '"--no-collector.timex"', - '"--no-collector.zfs"', - '"--collector.tcpstat"', - '"--collector.interrupts"', - '"--collector.meminfo_numa"', - '"--collector.processes"', - '"--collector.textfile"', - '"--collector.textfile.directory=/rootfs/textfiles"', - '"--collector.filesystem.ignored-mount-points"', - '"^/(sys|proc|dev|host|etc|run|run/lock|boot|var/lib/docker|run/docker/netns|var/lib/docker/aufs)($$|/)"', - ], - labels: [ - '"traefik.enable=true"', - '"traefik.http.routers.{{ node_exporter_id }}.service={{ node_exporter_id }}"', - '"traefik.http.routers.{{ node_exporter_id }}.rule=Host(`{{ inventory_hostname }}.{{ domain }}`)"', - '"traefik.http.routers.{{ node_exporter_id }}.entrypoints=monitoring-system"', - '"traefik.http.routers.{{ node_exporter_id }}.tls=true"', - '"traefik.http.routers.{{ node_exporter_id }}.tls.certresolver=letsencrypt"', - '"traefik.http.services.{{ node_exporter_id }}.loadbalancer.server.port={{ service_port_node_exporter }}"', - ], - volumes: [ - '"/proc:/host/proc:ro"', - '"/sys:/host/sys:ro"', - '"/:/rootfs:ro"', - ], - networks: [ - '"front-tier"' - ] - } - ] -} diff --git a/roles/prometheus_config/tasks/main.yml b/roles/prometheus_config/tasks/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/roles/traefik/vars/main.yml b/roles/traefik/vars/main.yml index 51cf2a3..2a6b17e 100644 --- a/roles/traefik/vars/main.yml +++ b/roles/traefik/vars/main.yml @@ -56,10 +56,6 @@ traefik_docker: { external: "0.0.0.0:{{ monitor_port_service }}", internal: "{{ monitor_port_service }}" }, - { - external: "0.0.0.0:{{ monitor_port_system }}", - internal: "{{ monitor_port_system }}" - }, { external: "0.0.0.0:{{ monitor_port_docker }}", internal: "{{ monitor_port_docker }}" @@ -79,4 +75,4 @@ traefik_docker: { ], } ] -} \ No newline at end of file +} diff --git a/tasks/autodiscover_pre_tasks.yml b/tasks/autodiscover_pre_tasks.yml index 111f10f..ff8b534 100644 --- a/tasks/autodiscover_pre_tasks.yml +++ b/tasks/autodiscover_pre_tasks.yml @@ -22,24 +22,6 @@ when: - debug -- name: "Reading hetzner server infos for stage <{{ stage }}>" - set_fact: - stage_server_infos: "{{ hetzner_servers.json.servers | json_query(querystr) | default([]) }}" - vars: - querystr: "[*].{ name: name, public_ip: public_net.ipv4.ip, private_ip: private_net[0].ip }" - delegate_to: 127.0.0.1 - become: false - tags: - - autodiscover - -- name: "Reading hetzner server infos for stage <{{ stage }}>" - debug: - msg: "{{ stage_server_infos }}" - delegate_to: 127.0.0.1 - become: false - tags: - - autodiscover - - name: "Reading hetzner private network infos for stage <{{ stage }}>" uri: url: "https://api.hetzner.cloud/v1/networks?label_selector=stage={{ stage }}" @@ -82,14 +64,17 @@ when: - debug -# TODO always match first private_net network is wrong -# use stage_private_network_id to match the right network -# expression [?network=`{{ stage_private_network_id }}`] does not work - name: "Reading private ip address for {{ inventory_hostname }}" set_fact: - stage_private_server_ip: "{{ hetzner_servers.json.servers | json_query(querystr) | first }}" - vars: - querystr: "[?name=='{{ inventory_hostname }}'].private_net[0].ip" + stage_private_server_ip: "{% for server in hetzner_servers.json.servers %}\ + {% if server.name == inventory_hostname %}\ + {% for private_net in server.private_net %}\ + {% if private_net.network == stage_private_network_id|int %}\ + {{ private_net.ip }}\ + {% endif %}\ + {% endfor %}\ + {% endif %}\ + {% endfor %}" delegate_to: 127.0.0.1 become: false tags: @@ -113,4 +98,33 @@ tags: - autodiscover when: - - debug \ No newline at end of file + - debug + +- name: "Reading hetzner server infos for stage <{{ stage }}>" + set_fact: + stage_server_infos: "\ + {% set list= [] %}\ + {% for server in hetzner_servers.json.servers %}\ + {% for private_net in server.private_net %}\ + {% if private_net.network == stage_private_network_id|int %}\ + {{ list.append({'service': server.labels.service, 'private_ip': private_net.ip, 'public_ip': server.public_net.ipv4.ip, 'name': server.name}) }}\ + {% endif %}\ + {% endfor %}\ + {% endfor %}\ + {{ list|list }}" + delegate_to: 127.0.0.1 + become: false + tags: + - autodiscover + when: + - debug + +- name: "Reading hetzner server infos for stage <{{ stage }}>" + debug: + msg: "{{ stage_server_infos }}" + delegate_to: 127.0.0.1 + become: false + tags: + - autodiscover + when: + - debug diff --git a/templates/prometheus/config/prometheus/prometheus.yml.j2 b/templates/prometheus/config/prometheus/prometheus.yml.j2 index ba45f8f..bd72ee6 100644 --- a/templates/prometheus/config/prometheus/prometheus.yml.j2 +++ b/templates/prometheus/config/prometheus/prometheus.yml.j2 @@ -223,7 +223,7 @@ scrape_configs: ############################################## - job_name: 'node-exporter' - scheme: {{ http_s }} + scheme: http metrics_path: '/metrics' static_configs: - targets: [ @@ -233,7 +233,7 @@ scrape_configs: and hostvars[host].node_exporter_enabled) or (hostvars[host].node_exporter_enabled is not defined) %} - '{{ host }}.{{ domain }}:{{ monitor_port_system }}', + '{{ (stage_server_infos | selectattr("name", "equalto", host) | first).private_ip }}:{{ monitor_port_system }}!{{ host }}.{{ domain }}', {% endif %} {% endfor %} ] @@ -245,9 +245,13 @@ scrape_configs: target_label: job replacement: 'node-exporter' - source_labels: [__address__] - regex: (.*):.* + regex: .*!(.*) target_label: instance replacement: $1 + - source_labels: [__address__] + regex: (.*)!.* + target_label: __address__ + replacement: $1 ############################################## ### Databases ####