DEV-729: fixing broken silencing of alerts for patchday; added rescue block to...

patchday.yml

@@ -9,35 +9,57 @@
     start: '{{ ansible_date_time.epoch }}'
 
   tasks:
-    - name: "Set VAR for silence start and end"
-      set_fact:
-        silence_starts_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime(start) }}"
-        silence_ends_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime( ( start | int ) + 3600 |int ) }}"
+    - name: "DO some stuff for silencing"
+      block:
+        - name: "Set VAR for silence start and end"
+          set_fact:
+            silence_starts_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime(start) }}"
+            silence_ends_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime( ( start | int ) + 3600 |int ) }}"
+
+        - name: "Set VAR - define prometheus silence object"
+          set_fact:
+            silence:
+              matchers:
+                - name: job
+                  value: .+
+                  isRegex: true
+              startsAt: '{{ silence_starts_at }}'
+              endsAt: '{{ silence_ends_at }}'
+              createdBy: patchday-automatism
+              comment: patchday
+              id:
+
+        - name: "Schedule silences for stage..."
+          uri:
+            url: "https://{{ stage }}-prometheus-01-alertmanager.smardigo.digital/api/v2/silences"
+            url_username: "{{ alertmanager_admin_username }}"
+            url_password: "{{ alertmanager_admin_password }}"
+            method: POST
+            status_code: [200]
+            headers:
+              Content-Type: application/json
+            body_format: json
+            body: '{{ silence | to_json }}'
+
+      rescue:
+        - name: "Rescue silencing - sending mail to DEVOPS-DL"
+          delegate_to: '{{ stage }}-mail-01'
+          community.general.mail:
+            host: localhost
+            port: 25
+            to: '{{ devops_email_address }}'
+            subject: "patchday( {{ lookup('pipe','date +%Y-%m-%d_%H:%M') }} ) problem report for failed silencing"
+            body: |
+              Dear Sir or Madam,
+
+              silencing alerts for patchday failed.
+
+              Plz check what happened/ fix it little padawan ;)
+
+              kind regards,
+
+              your automation-bofh
 
-    - name: "Set VAR - define prometheus silence object"
-      set_fact:
-        silence:
-          matchers:
-            - name: job
-              value: .+
-              isRegex: true
-          startsAt: '{{ silence_starts_at }}'
-          endsAt: '{{ silence_ends_at }}'
-          createdBy: patchday-automatism
-          comment: patchday
-          id:
-
-    - name: "Schedule silences for stage..."
-      uri:
-        url: "https://{{ stage }}-prometheus-01-alertmanager.smardigo.digital/api/v2/silences"
-        method: POST
-        status_code: [200]
-        headers:
-          Content-Type: application/json
-        body_format: json
-        body: '{{ silence | to_json }}'
-      ignore_errors: yes # noqa ignore-errors
-      # if failing, patchday continues
 
 - hosts: elastic
   serial: 1