DEV-580: Added prom2teams alert and receiver for email

roles/prometheus/vars/main.yml

@@ -126,6 +126,16 @@ prometheus_docker: {
       name: "{{ prom2teams_id }}",
       image_name: "{{ prom2teams_image_name }}",
       image_version: "{{ prom_prom2teams_version }}",
+      environment: [
+        "PROM2TEAMS_PROMETHEUS_METRICS: \"true\"",
+        "DEBUG_METRICS: \"true\"",
+      ],
+      ports: [
+        {
+          "external": "8089",
+          "internal": "8089",
+        },
+      ],
       volumes: [
         '"./config/prom2teams/config.ini:/opt/prom2teams/config.ini"',
       ],

templates/prometheus/config/alertmanager/config.yml.j2

@@ -8,9 +8,22 @@ route:
     - receiver: 'netgo_msteams_receiver'
       match:
         stage: '{{ stage }}'
+    - receiver: 'netgo_email_receiver'
+      match:
+        receiver: email
 
 receivers:
 - name: 'netgo_msteams_receiver'
   webhook_configs:
     - send_resolved: true
       url: 'http://{{ prom2teams_id }}:8089/v2/NetgoTeamsConnector'
+
+- name: 'netgo_email_receiver'
+  email_configs:
+  - to: '{{ devops_email_address }}'
+    from: 'prometheus_{{ stage }}@smardigo.digital'
+    smarthost: '{{ shared_service_mail_hostname }}:25'
+    auth_username: ''
+    auth_identity: ''
+    auth_password: ''
+    require_tls: false

templates/prometheus/config/prometheus/alert.rules.j2

@@ -503,3 +503,13 @@ groups:
       identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
       summary: "hetzner locked server exists"
       description: "hetzner locked server exists"
+
+  - alert: prom2teams down
+    expr: process_cpu_seconds_total{application="prom2teams"} <= 0 or absent(process_cpu_seconds_total{application="prom2teams"})
+    for: 5m
+    labels:
+      severity: critical
+      receiver: email
+    annotations:
+      summary: "prom2teams is down"
+      description: "prom2teams is down"

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -57,6 +57,18 @@ scrape_configs:
         target_label:  instance
         replacement:   '{{ inventory_hostname }}-alertmanager.{{ domain }}'
 
+  - job_name: 'prom2teams'
+    scheme: http
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: [
+          '{{ inventory_hostname }}-prom2teams:8089'
+        ]
+        labels:
+          env: {{ stage }}
+          project: monitoring
+          application: prom2teams
+
   - job_name: 'blackbox'
     metrics_path: /probe
     params: