From 7d211d93adb4fb9dafbb33c3ecdb16cdb043b8f2 Mon Sep 17 00:00:00 2001
From: Sven Ketelsen <sven.ketelsen@netgo.de>
Date: Thu, 27 Jan 2022 21:44:03 +0100
Subject: [PATCH] bugfix: prometheus update_config didn't reload config

- container isn't restarted always anymore
- added reload by rest api
---
 roles/prometheus/tasks/_update_config.yml |  1 +
 roles/prometheus/tasks/main.yml           | 53 +++++++++++++----------
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/roles/prometheus/tasks/_update_config.yml b/roles/prometheus/tasks/_update_config.yml
index 796e3da..af9ab12 100644
--- a/roles/prometheus/tasks/_update_config.yml
+++ b/roles/prometheus/tasks/_update_config.yml
@@ -17,4 +17,5 @@
   uri:
     url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}/-/reload"
     method: POST
+    timeout: 300 # blocks until reload configuration is complete
     status_code: 200
diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml
index fb89026..e05846e 100644
--- a/roles/prometheus/tasks/main.yml
+++ b/roles/prometheus/tasks/main.yml
@@ -4,29 +4,17 @@
 ###   update_config
 ###   update_deployment
 
-- name: "Setup DNS configuration for {{ inventory_hostname }} prometheus"
+- name: "Setup DNS configuration for <{{ inventory_hostname }}>"
   include_role:
     name: sma_digitalocean
     tasks_from: domain
   vars:
     record_data: "{{ stage_server_ip }}"
-    record_name: "{{ inventory_hostname }}-prometheus"
-
-- name: "Setup DNS configuration for {{ inventory_hostname }} grafana"
-  include_role:
-    name: sma_digitalocean
-    tasks_from: domain
-  vars:
-    record_data: "{{ stage_server_ip }}"
-    record_name: "{{ inventory_hostname }}-grafana"
-
-- name: "Setup DNS configuration for {{ inventory_hostname }} alertmanager"
-  include_role:
-    name: sma_digitalocean
-    tasks_from: domain
-  vars:
-    record_data: "{{ stage_server_ip }}"
-    record_name: "{{ inventory_hostname }}-alertmanager"
+    record_name: "{{ item }}"
+  loop:
+    - "{{ prometheus_id }}"
+    - "{{ grafana_id }}"
+    - "{{ alertmanager_id }}"
 
 - name: "Check if {{ inventory_hostname }}/docker-compose.yml exists"
   stat:
@@ -56,7 +44,7 @@
     current_group: "{{ docker_group }}"
     current_docker: "{{ prometheus_docker }}"
   tags:
-    - update_config
+    - update_deployment
 
 - name: "Deploy service templates for {{ inventory_hostname }}"
   include_role:
@@ -77,12 +65,33 @@
     state: present
     pull: yes
   tags:
-    - update_config
     - update_deployment
 
+- name: "Wait for <{{ http_s }}://{{ prometheus_id }}.{{ domain }}>"
+  uri:
+    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}"
+    method: GET
+    status_code: 200
+    return_content: yes
+  register: prometheus_stats
+  until: prometheus_stats.status == 200
+  retries: 10
+  delay: 60
+  tags:
+    - update_config
+
+- name: "Reload prometheus configuration"
+  uri:
+    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}/-/reload"
+    method: POST
+    timeout: 300 # blocks until reload configuration is complete
+    status_code: 200
+  tags:
+    - update_config
+
 - name: "Wait for {{ http_s }}://{{ inventory_hostname }}-grafana.{{ domain }}"
   uri:
-    url: "{{ http_s }}://{{ inventory_hostname }}-grafana.{{ domain }}/api/admin/stats"
+    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}/api/admin/stats"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     force_basic_auth: yes
@@ -96,7 +105,7 @@
 
 - name: Create grafana users
   uri:
-    url: "{{ http_s }}://{{ inventory_hostname }}-grafana.{{ domain }}/api/admin/users"
+    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}/api/admin/users"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     force_basic_auth: yes