diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 3ccbd64..9f1d77c 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -150,3 +150,22 @@ job: /root/do_too_many_req_metrics.sh tags: - update-do-metrics + +- name: "Create hetzner-metrics script from template" + template: + src: hetzner_unattached_volumes.py.j2 + dest: /root/hetzner_unattached_volumes.py + mode: 0700 + owner: root + group: root + tags: + - update-hetzner-metrics + +- name: "Create cronjob to exec hetzner-metrics script" + ansible.builtin.cron: + name: "update hetzner metrics" + minute: "*" + job: /root/hetzner_unattached_volumes.py + tags: + - update-hetzner-metrics + diff --git a/roles/prometheus/templates/hetzner_unattached_volumes.py.j2 b/roles/prometheus/templates/hetzner_unattached_volumes.py.j2 new file mode 100644 index 0000000..3261b38 --- /dev/null +++ b/roles/prometheus/templates/hetzner_unattached_volumes.py.j2 @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import requests + + +access_token = '{{ hetzner_authentication_ansible_vault }}' + +node_exporter_txt_dir = '/var/lib/prometheus/node-exporter' +metrics_file = node_exporter_txt_dir + "/hetzner_metrics.prom" + +query= {'per_page': '1000'} +headers = {'Authorization': 'Bearer ' + access_token} + +try: + response_volumes = requests.get("https://api.hetzner.cloud/v1/volumes", headers=headers, params=query) + response_servers = requests.get("https://api.hetzner.cloud/v1/servers", headers=headers, params=query) +except requests.exceptions.RequestException as e: # This is the correct syntax + f = open(metrics_file, "w") + f.write("\n") + f.close() + raise SystemExit(e) + + +if response_volumes.ok and response_servers.ok: + + + volume_json = response_volumes.json()["volumes"] + unattached_volume_count = 0 + for x in range(len(volume_json)): + if volume_json[x]["server"] == None: + unattached_volume_count+=1 + + locked_servers_count = 0 + servers_json = response_servers.json()["servers"] + for x in range(len(servers_json)): + if servers_json[x]["locked"] == "true": + locked_servers_count+=1 + + ratelimit_limit = response_servers.headers['ratelimit-limit'] + ratelimit_remaining = response_servers.headers['ratelimit-remaining'] + + + f = open(metrics_file, "w") + f.write( + "hetzner_api_ratelimit_remaining " + str(ratelimit_remaining) + "\n" + + "hetzner_api_ratelimit_limit " + str(ratelimit_limit) + "\n" + + "hetzner_api_unattached_volumes " + str(unattached_volume_count) + "\n" + + "hetzner_api_locked_servers " + str(locked_servers_count) + "\n" + ) + f.close() +else: + f = open(metrics_file, "w") + f.write("\n") + f.close() \ No newline at end of file diff --git a/templates/prometheus/config/prometheus/alert.rules.j2 b/templates/prometheus/config/prometheus/alert.rules.j2 index 3041f40..0e2a9dc 100644 --- a/templates/prometheus/config/prometheus/alert.rules.j2 +++ b/templates/prometheus/config/prometheus/alert.rules.j2 @@ -360,4 +360,35 @@ groups: summary: "postgres backup zombies, have not been deleted" description: "postgres backup zombies, have not been deleted" + - alert: hetzner unattached volumes + expr: hetzner_api_unattached_volumes > 0 or absent(hetzner_api_unattached_volumes) + for: 2h + labels: + severity: critical + annotations: + identifier: '{{ '{{' }} $labels.instance {{ '}}' }}' + summary: "unattached volumes in hetzner" + description: "unattached volumes in hetzner" + + + - alert: hetzner ratelimit_remaining low + expr: hetzner_api_ratelimit_remaining < 720 or absent(hetzner_api_ratelimit_remaining) + for: 1h + labels: + severity: critical + annotations: + identifier: '{{ '{{' }} $labels.instance {{ '}}' }}' + summary: "hetzner ratelimit_remaining below 720" + description: "hetzner ratelimit_remaining below 720" + + - alert: hetzner locked server exists + expr: hetzner_api_locked_servers > 0 or absent(hetzner_api_locked_servers) + for: 1h + labels: + severity: critical + annotations: + identifier: '{{ '{{' }} $labels.instance {{ '}}' }}' + summary: "hetzner locked server exists" + description: "hetzner locked server exists" +