NOTICKET: abolishing nightly false positive alerts

templates/prometheus/config/prometheus/alert.rules.j2

@@ -335,19 +335,19 @@ groups:
 
   - alert: offsite backup pending
     expr: |
-      (time() - offsite_backup_archive_started_seconds) > 27 * 3600 or
-      (time() - offsite_backup_archive_ended_seconds) > 27 * 3600 or
-      (time() - offsite_backup_transfer_started_seconds) > 27 * 3600 or
-      (time() - offsite_backup_transfer_ended_seconds) > 27 * 3600 or
-      (time() - offsite_backup_forget_started_seconds) > 27 * 3600 or
-      (time() - offsite_backup_forget_ended_seconds) > 27 * 3600
+      (time() - offsite_backup_archive_started_seconds) > 30 * 3600 or
+      (time() - offsite_backup_archive_ended_seconds) > 30 * 3600 or
+      (time() - offsite_backup_transfer_started_seconds) > 30 * 3600 or
+      (time() - offsite_backup_transfer_ended_seconds) > 30 * 3600 or
+      (time() - offsite_backup_forget_started_seconds) > 30 * 3600 or
+      (time() - offsite_backup_forget_ended_seconds) > 30 * 3600
     for: 1m
     labels:
       severity: critical
     annotations:
       identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
-      summary: "one or more offsite backup metric timestamps for Instance <{{ '{{' }} $labels.instance {{ '}}' }}> older then 27h"
-      description: "offsite backups older then 27h."
+      summary: "one or more offsite backup metric timestamps for Instance <{{ '{{' }} $labels.instance {{ '}}' }}> older then 30h"
+      description: "offsite backups older then 30h."
 
   - alert: offsite backup metrics unavailable
     expr: |