DEV-729: fixing broken silencing of alerts for patchday; added rescue block to...

patchday.yml

@@ -9,6 +9,8 @@
     start: '{{ ansible_date_time.epoch }}'
 
   tasks:
+    - name: "DO some stuff for silencing"
+      block:
         - name: "Set VAR for silence start and end"
           set_fact:
             silence_starts_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime(start) }}"
@@ -30,14 +32,34 @@
         - name: "Schedule silences for stage..."
           uri:
             url: "https://{{ stage }}-prometheus-01-alertmanager.smardigo.digital/api/v2/silences"
+            url_username: "{{ alertmanager_admin_username }}"
+            url_password: "{{ alertmanager_admin_password }}"
             method: POST
             status_code: [200]
             headers:
               Content-Type: application/json
             body_format: json
             body: '{{ silence | to_json }}'
-      ignore_errors: yes # noqa ignore-errors
+
-      # if failing, patchday continues
+      rescue:
+        - name: "Rescue silencing - sending mail to DEVOPS-DL"
+          delegate_to: '{{ stage }}-mail-01'
+          community.general.mail:
+            host: localhost
+            port: 25
+            to: '{{ devops_email_address }}'
+            subject: "patchday( {{ lookup('pipe','date +%Y-%m-%d_%H:%M') }} ) problem report for failed silencing"
+            body: |
+              Dear Sir or Madam,
+
+              silencing alerts for patchday failed.
+
+              Plz check what happened/ fix it little padawan ;)
+
+              kind regards,
+
+              your automation-bofh
+
 
 - hosts: elastic
   serial: 1