Dev 549 alert unused volumes

roles/prometheus/tasks/main.yml

@@ -150,3 +150,22 @@
     job: /root/do_too_many_req_metrics.sh 
   tags:
     - update-do-metrics
+
+- name: "Create hetzner-metrics script from template" 
+  template:
+    src: hetzner_unattached_volumes.py.j2
+    dest: /root/hetzner_unattached_volumes.py
+    mode: 0700
+    owner: root
+    group: root 
+  tags:
+    - update-hetzner-metrics
+
+- name: "Create cronjob to exec hetzner-metrics script"
+  ansible.builtin.cron:
+    name: "update hetzner metrics"
+    minute: "*"
+    job: /root/hetzner_unattached_volumes.py
+  tags:
+    - update-hetzner-metrics
+

roles/prometheus/templates/hetzner_unattached_volumes.py.j2

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import requests
+
+
+access_token = '{{ hetzner_authentication_ansible_vault }}'
+
+node_exporter_txt_dir = '/var/lib/prometheus/node-exporter'
+metrics_file = node_exporter_txt_dir + "/hetzner_metrics.prom"
+
+query= {'per_page': '1000'}
+headers = {'Authorization': 'Bearer ' + access_token}
+
+try:
+  response_volumes = requests.get("https://api.hetzner.cloud/v1/volumes", headers=headers, params=query)
+  response_servers = requests.get("https://api.hetzner.cloud/v1/servers", headers=headers, params=query)
+except requests.exceptions.RequestException as e:  # This is the correct syntax
+  f = open(metrics_file, "w")
+  f.write("\n")
+  f.close()
+  raise SystemExit(e)
+
+
+if response_volumes.ok and response_servers.ok:
+  
+  
+  volume_json = response_volumes.json()["volumes"]
+  unattached_volume_count = 0
+  for x in range(len(volume_json)):
+    if volume_json[x]["server"] == None:
+      unattached_volume_count+=1
+
+  locked_servers_count = 0
+  servers_json = response_servers.json()["servers"]
+  for x in range(len(servers_json)):
+    if servers_json[x]["locked"] == "true":
+      locked_servers_count+=1
+
+  ratelimit_limit = response_servers.headers['ratelimit-limit']
+  ratelimit_remaining = response_servers.headers['ratelimit-remaining']
+  
+  
+  f = open(metrics_file, "w")
+  f.write(
+  "hetzner_api_ratelimit_remaining " + str(ratelimit_remaining) + "\n" 
+  + "hetzner_api_ratelimit_limit " + str(ratelimit_limit) + "\n" 
+  + "hetzner_api_unattached_volumes " + str(unattached_volume_count) + "\n"
+  + "hetzner_api_locked_servers " + str(locked_servers_count) + "\n"
+  )
+  f.close()
+else:
+  f = open(metrics_file, "w")
+  f.write("\n")
+  f.close()

templates/prometheus/config/prometheus/alert.rules.j2

@@ -360,4 +360,35 @@ groups:
       summary: "postgres backup zombies, have not been deleted"
       description: "postgres backup zombies, have not been deleted"
 
+  - alert: hetzner unattached volumes
+    expr: hetzner_api_unattached_volumes > 0 or absent(hetzner_api_unattached_volumes)
+    for: 2h
+    labels:
+      severity: critical
+    annotations:
+      identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
+      summary: "unattached volumes in hetzner"
+      description: "unattached volumes in hetzner"
+
+
+  - alert: hetzner ratelimit_remaining low
+    expr: hetzner_api_ratelimit_remaining < 720 or absent(hetzner_api_ratelimit_remaining)
+    for: 1h
+    labels:
+      severity: critical
+    annotations:
+      identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
+      summary: "hetzner ratelimit_remaining below 720"
+      description: "hetzner ratelimit_remaining below 720"
+
+  - alert: hetzner locked server exists
+    expr: hetzner_api_locked_servers > 0 or absent(hetzner_api_locked_servers)
+    for: 1h
+    labels:
+      severity: critical
+    annotations:
+      identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
+      summary: "hetzner locked server exists"
+      description: "hetzner locked server exists"
+