DEV-1078: added alert prom2teams down

clusterspecifics/alerts/alert_prometheus_msteams.yaml

@@ -0,0 +1,22 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: kube-prometheus-stack-prometheus    # dieser name ist fest
+    role: alert-rules                               # auch fest
+    release: kube-prometheus-stack                  # IMPORTANT: ohne das Label wird die Rule vom PrometheusOperator nicht gefunden
+  name: prometheus-msteams-alert
+  namespace: monitoring              # Rule im namespace anlegen in dem die zu ueberwachende App laeuft
+spec:
+  groups:
+    - name: msteams_alerts
+      rules:
+        - alert: PrometheusMSTeamsDown
+          expr: up{job="prometheus-msteams"} == 0
+          for: 5m
+          labels:
+            severity: critical
+            receiver: email
+          annotations:
+            summary: "Prometheus MSTeams is down"
+            description: "The prometheus-msteams service has been down for more than 5 minutes."

clusterspecifics/alerts/kustomization.yaml

@@ -1,3 +1,4 @@
 resources:
 - alert_hetzner_api_requestlimit.yaml
 - alert_backup_minio.yaml
+- alert_prometheus_msteams.yaml

templates/secret_alertmanager-config.yaml

@@ -41,7 +41,11 @@ stringData:
         receiver: alerting-nso-devops
         matchers:
           - severity = critical
-        continue: false
+        continue: true
+      - receiver: 'netgo_email_receiver'
+        match:
+          receiver: email
+        continue: true
     inhibit_rules:
       - source_match:
           alertname: "Watchdog"
@@ -63,6 +67,15 @@ stringData:
       webhook_configs:
       - url: "http://prometheus-msteams:2000/alert-nso-team-erechnung-mobene"
         send_resolved: true
+    - name: netgo_email_receiver
+      email_configs:
+      - to: 'nso.devops@netgo.de'
+        from: 'prometheus_prodwork01@netgo.de'
+        smarthost: '10.3.0.11:25'
+        auth_username: ''
+        auth_identity: ''
+        auth_password: ''
+        require_tls: false
 
 kind: Secret
 metadata: