diff --git a/clusterspecifics/alerts/alert_prometheus_msteams.yaml b/clusterspecifics/alerts/alert_prometheus_msteams.yaml new file mode 100644 index 0000000..70d0fca --- /dev/null +++ b/clusterspecifics/alerts/alert_prometheus_msteams.yaml @@ -0,0 +1,22 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + prometheus: kube-prometheus-stack-prometheus # dieser name ist fest + role: alert-rules # auch fest + release: kube-prometheus-stack # IMPORTANT: ohne das Label wird die Rule vom PrometheusOperator nicht gefunden + name: prometheus-msteams-alert + namespace: monitoring # Rule im namespace anlegen in dem die zu ueberwachende App laeuft +spec: + groups: + - name: msteams_alerts + rules: + - alert: PrometheusMSTeamsDown + expr: up{job="prometheus-msteams"} == 0 + for: 5m + labels: + severity: critical + receiver: email + annotations: + summary: "Prometheus MSTeams is down" + description: "The prometheus-msteams service has been down for more than 5 minutes." \ No newline at end of file diff --git a/clusterspecifics/alerts/kustomization.yaml b/clusterspecifics/alerts/kustomization.yaml index e8033d3..cf7de4a 100644 --- a/clusterspecifics/alerts/kustomization.yaml +++ b/clusterspecifics/alerts/kustomization.yaml @@ -1,3 +1,4 @@ resources: - alert_hetzner_api_requestlimit.yaml -- alert_backup_minio.yaml \ No newline at end of file +- alert_backup_minio.yaml +- alert_prometheus_msteams.yaml \ No newline at end of file diff --git a/templates/secret_alertmanager-config.yaml b/templates/secret_alertmanager-config.yaml index 191d149..a43040a 100644 --- a/templates/secret_alertmanager-config.yaml +++ b/templates/secret_alertmanager-config.yaml @@ -41,7 +41,11 @@ stringData: receiver: alerting-nso-devops matchers: - severity = critical - continue: false + continue: true + - receiver: 'netgo_email_receiver' + match: + receiver: email + continue: true inhibit_rules: - source_match: alertname: "Watchdog" @@ -63,6 +67,15 @@ stringData: webhook_configs: - url: "http://prometheus-msteams:2000/alert-nso-team-erechnung-mobene" send_resolved: true + - name: netgo_email_receiver + email_configs: + - to: 'nso.devops@netgo.de' + from: 'prometheus_prodwork01@netgo.de' + smarthost: '10.3.0.11:25' + auth_username: '' + auth_identity: '' + auth_password: '' + require_tls: false kind: Secret metadata: