From a9eea5026a43832163effa96426493f9e17a6c91 Mon Sep 17 00:00:00 2001 From: friedrich goerz Date: Wed, 10 May 2023 16:16:27 +0200 Subject: [PATCH] DEV-1029: added alert to get notified in case of api error above threshold --- clusterspecifics/alerts/alert_backup_minio.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/clusterspecifics/alerts/alert_backup_minio.yaml b/clusterspecifics/alerts/alert_backup_minio.yaml index 1321ba5..6ce94fa 100644 --- a/clusterspecifics/alerts/alert_backup_minio.yaml +++ b/clusterspecifics/alerts/alert_backup_minio.yaml @@ -14,7 +14,7 @@ spec: - alert: backup_minio_nsodev_successful for: 1m labels: - severity: warning # Sollte dazu geschrieben werden um Alerts Sauber zwischen Teams zu trennen + severity: warning expr: nightly_backup_successful_nsodev{stage="nsodev"} > 0 # hier kommt die eigentliche Alert expression rein annotations: message: Nightly Backup of Minio NSODEV Bucket failed @@ -22,7 +22,7 @@ spec: - alert: backup_minio_cusqa_successful for: 1m labels: - severity: warning # Sollte dazu geschrieben werden um Alerts Sauber zwischen Teams zu trennen + severity: warning expr: nightly_backup_successful_cusqa{stage="cusqa"} > 0 # hier kommt die eigentliche Alert expression rein annotations: message: Nightly Backup of Minio CUSQA Bucket failed @@ -30,7 +30,16 @@ spec: - alert: backup_minio_cusprod_successful for: 1m labels: - severity: warning # Sollte dazu geschrieben werden um Alerts Sauber zwischen Teams zu trennen + severity: warning expr: nightly_backup_successful_cusprod{stage="cusprod"} > 0 # hier kommt die eigentliche Alert expression rein annotations: - message: Nightly Backup of Minio CUSPROD Bucket failed \ No newline at end of file + message: Nightly Backup of Minio CUSPROD Bucket failed + - name: "minio-api.rules" # ".rules" muss unbedingt im Namen enthalten sein!! + rules: + - alert: api_errors + for: 5m + labels: + severity: critical + expr: sum by (server,api) (increase(minio_s3_requests_errors_total{job="minio"}[60m])) > 50 + annotations: + message: S3(minio) API error above threshold of 50. Plz check. \ No newline at end of file