DEV-1200 Enable Prometheus Remote Write for qa and prod

group_vars/all/services.yml

@@ -17,6 +17,8 @@ shared_service_url_management: "https://{{ shared_service_hostname_management }}
 shared_service_hostname_management: "{{ shared_service_host_management }}-connect.{{ domain_env }}"
 shared_service_url_pgadmin4: "https://{{ shared_service_hostname_pgadmin4 }}"
 shared_service_hostname_pgadmin4: "{{ stage }}-pgadmin4-01-pgadmin4.{{ domain_env }}"
+shared_service_url_grafana: "https://{{ shared_service_hostname_grafana }}"
+shared_service_hostname_grafana: "{{ stage }}-prometheus-01-grafana.{{ domain_env }}"
 
 shared_service_hostname_logstash: "{{ stage }}-elastic-stack-logstash-01"
 
@@ -34,6 +36,8 @@ shared_service_kube_url_kibana: "https://{{ shared_service_kube_hostname_kibana
 shared_service_kube_hostname_kibana: "{{ stage_kube }}-kibana.{{ domain_env }}"
 shared_service_kube_url_prometheus: "https://{{ shared_service_kube_hostname_prometheus }}"
 shared_service_kube_hostname_prometheus: "{{ stage_kube }}-prometheus.{{ domain_env }}"
+shared_service_kube_url_grafana: "https://{{ shared_service_kube_hostname_grafana }}"
+shared_service_kube_hostname_grafana: "{{ stage_kube }}-grafana.{{ domain_env }}"
 shared_service_kube_jaeger_collector_hostname: "{{ stage_kube }}-jaeger-collector.{{ domain_env }}"
 
 shared_service_kube_loadbalancer_public_ip_not_available: "public loadbalancer ip not available"
@@ -58,6 +62,8 @@ shared_service_default_additional_hosts:
     ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_hostname_prometheus }}"
     ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
+  - name: "{{ shared_service_kube_hostname_grafana }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_jaeger_collector_hostname }}"
     ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
 

group_vars/stage_devnso/plain.yml

@@ -9,6 +9,8 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 
+shared_service_url_grafana: "{{ shared_service_kube_url_grafana }}"
+
 shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain_env }}"
 shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain_env }}"
 

group_vars/stage_devnso/prometheus.yml

@@ -1,4 +1,8 @@
 ---
 prometheus_tsdb_rentention_time: "2w"
 
-prometheus_remote_write_enabled: false
+prometheus_federation_enabled: false
+
+prometheus_alertmanager_enabled: false
+prometheus_prom2teams_enabled: false
+prometheus_grafana_enabled: false

group_vars/stage_prodnso/plain.yml

@@ -9,6 +9,8 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 
+shared_service_url_grafana: "{{ shared_service_kube_url_grafana }}"
+
 shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain_env }}"
 shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain_env }}"
 

group_vars/stage_prodnso/prometheus.yml

@@ -10,4 +10,8 @@ smardigo_connect_extra_servers: []
 # check firewall settings
 node_exporter_extra_servers: []
 
-prometheus_remote_write_enabled: false
+prometheus_federation_enabled: false
+
+prometheus_alertmanager_enabled: false
+prometheus_prom2teams_enabled: false
+prometheus_grafana_enabled: false

group_vars/stage_qanso/plain.yml

@@ -9,6 +9,8 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 
+shared_service_url_grafana: "{{ shared_service_kube_url_grafana }}"
+
 shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain_env }}"
 shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain_env }}"
 

group_vars/stage_qanso/prometheus.yml

@@ -1,4 +1,8 @@
 ---
 prometheus_tsdb_rentention_time: "2w"
 
-prometheus_remote_write_enabled: false
+prometheus_federation_enabled: false
+
+prometheus_alertmanager_enabled: false
+prometheus_prom2teams_enabled: false
+prometheus_grafana_enabled: false

roles/prometheus/tasks/main.yml

@@ -70,9 +70,10 @@
   tags:
     - update_config
 
-- name: Create or update Grafana users
+- name: "Create or update Grafana users"
+  become: false
   community.grafana.grafana_user:
-    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}"
+    url: "{{ shared_service_url_grafana }}"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     name: "{{ item.name }}"
@@ -84,12 +85,12 @@
   loop: "{{ grafana_users }}"
   tags:
     - grafana-user-update
-  when:
-    - prometheus_grafana_enabled
+  delegate_to: localhost
 
-- name: "Get all Dashboard uids from {{ http_s }}://{{ inventory_hostname }}-grafana.{{ domain }}"
+- name: "Get all Dashboard uids from {{ shared_service_url_grafana }}"
+  become: false
   uri:
-    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}/api/search"
+    url: "{{ shared_service_url_grafana }}/api/search"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     force_basic_auth: yes
@@ -102,12 +103,11 @@
   delay: 60
   tags:
     - grafana-user-update
-  when:
-    - prometheus_grafana_enabled
+  delegate_to: localhost
 
 - name: "Get all existing Dashboard uids"
   set_fact:
-    grafana_dashboards: "{{ grafana_dashboards_plain.json | json_query('[].{uid: uid, type: type, title: title}') if prometheus_grafana_enabled else [] }}"
+    grafana_dashboards: "{{ grafana_dashboards_plain.json | json_query('[].{uid: uid, type: type, title: title}') }}"
   tags:
     - grafana-user-update
 
@@ -118,11 +118,11 @@
     - grafana-user-update
   when:
     - debug
-    - prometheus_grafana_enabled
 
-- name: Restrict admin dashboard permissions
+- name: "Restrict admin dashboard permissions"
+  become: false
   uri:
-    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}/api/dashboards/uid/{{ item.uid }}/permissions"
+    url: "{{ shared_service_url_grafana }}/api/dashboards/uid/{{ item.uid }}/permissions"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     force_basic_auth: yes
@@ -139,13 +139,14 @@
   when:
     - item.type == 'dash-db'
     - item.uid not in grafana_dashboard_whitelist
-    - prometheus_grafana_enabled
   tags:
     - grafana-user-update
+  delegate_to: localhost
 
-- name: Allow viewer dashboard permissions
+- name: "Allow viewer dashboard permissions"
+  become: false
   uri:
-    url: "{{ http_s }}://{{ grafana_id }}.{{ domain }}/api/dashboards/uid/{{ item.uid }}/permissions"
+    url: "{{ shared_service_url_grafana }}/api/dashboards/uid/{{ item.uid }}/permissions"
     url_username: "{{ grafana_admin_username }}"
     url_password: "{{ grafana_admin_password }}"
     force_basic_auth: yes
@@ -162,9 +163,9 @@
   when:
     - item.type == 'dash-db'
     - item.uid in grafana_dashboard_whitelist
-    - prometheus_grafana_enabled
   tags:
     - grafana-user-update
+  delegate_to: localhost
 
 - name: "Create digitalocean api metric script from template"
   template: