DEV-1060 Prometheus Setup für DEMO MPMX anpassen (Metriken/Alerts)

group_vars/all/elastic.yml

@@ -0,0 +1,2 @@
+---
+elastic_cluster_settings_max_shards: 1000

group_vars/all/plain.yml

@@ -1,5 +1,4 @@
 ---
-
 ansible_ssh_host: "{{ stage_server_domain }}"
 
 ssh_macs:
@@ -59,8 +58,6 @@ stage_server_domain: "{{ inventory_hostname }}.{{ domain }}"
 stage_server_url: "{{ http_s }}://{{ stage_server_domain }}"
 stage_kube_load_balancer: "{{ stage_kube }}-ingress"
 
-alertmanager_channel_smardigo: "#monitoring-{{ stage }}"
-
 hetzner_server_type: cx11
 hetzner_server_image: ubuntu-20.04
 hetzner_location: nbg1

group_vars/all/prometheus.yml

@@ -6,15 +6,33 @@ node_exporter_listen_address: "{{ stage_private_server_ip }}"
 blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
 
 blackbox_http_2xx_targets:
-- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
-- 'https://{{ stage_kube }}-awx.smardigo.digital'
+  - "https://{{ stage }}-keycloak-01.smardigo.digital/auth/"
+  - "https://{{ stage_kube }}-awx.smardigo.digital"
 # - 'https://{{ stage }}-management-01-connect.smardigo.digital/'
+
 blackbox_http_2xx_additional_targets: []
 
+prometheus_tsdb_rentention_time: "4w"
+
 prometheus_federation_enabled: true
-kubernetes_prometheus_endpoint: "{{ stage_kube }}-prometheus.{{ domain }}"
+prometheus_remote_write_enabled: true
 
 prometheus_alert_diskspaceusage_warning: 85
 prometheus_alert_pg_replication_lag: 120
 
-elastic_cluster_settings_max_shards: 1000
+alertmanager_channel_smardigo: "#monitoring-{{ stage }}"
+
+prometheus_enabled: true
+prometheus_alertmanager_enabled: true
+prometheus_grafana_enabled: true
+prometheus_prom2teams_enabled: true
+
+prometheus_service_names:
+  - "{{ (prometheus_enabled | default(true)) | ternary(prometheus_id, '') }}"
+  - "{{ (prometheus_alertmanager_enabled | default(true)) | ternary(alertmanager_id, '') }}"
+  - "{{ (prometheus_grafana_enabled | default(true)) | ternary(grafana_id, '') }}"
+  - "{{ (prometheus_prom2teams_enabled | default(true)) | ternary(prom2teams_id, '') }}"
+
+prometheus_docker_volume_names:
+  - "{{ (prometheus_alertmanager_enabled | default(true)) | ternary(alertmanager_id + '-data', '') }}"
+  - "{{ (prometheus_grafana_enabled | default(true)) | ternary(grafana_id + '-data', '') }}"

group_vars/stage_demompmx/prometheus.yml

@@ -7,6 +7,9 @@ alertmanager_admin_username: "alertmanager-admin"
 alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
 alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
 
-prometheus_tsdb_rentention_time: '2w'
 # federation for k8s prometheus -> stage prometheus
 prometheus_federation_enabled: false
+
+prometheus_alertmanager_enabled: false
+prometheus_prom2teams_enabled: false
+prometheus_grafana_enabled: false

group_vars/stage_dev/prometheus.yml

@@ -1,3 +1,2 @@
 ---
-prometheus_tsdb_rentention_time: '2w'
-kubernetes_prometheus_endpoint: "{{ stage_kube }}-prometheus.{{ domain }}"
+prometheus_remote_write_enabled: false

group_vars/stage_prodnso/elastic.yml

@@ -0,0 +1,2 @@
+---
+elastic_cluster_settings_max_shards: 1500

group_vars/stage_prodnso/prometheus.yml

@@ -2,7 +2,7 @@
 prometheus_lvm_hcloudvol_size: 30
 prometheus_lvm_hcloudvol_count: 10
 
-prometheus_tsdb_rentention_time: '90d'
+prometheus_tsdb_rentention_time: "90d"
 
 # check firewall settings
 smardigo_connect_extra_servers:
@@ -12,6 +12,4 @@ smardigo_connect_extra_servers:
 node_exporter_extra_servers:
   - ext-bdev-mpmexec-02.smardigo.digital
 
-
-kubernetes_prometheus_endpoint: "prodnso-prometheus.{{ domain }}"
-elastic_cluster_settings_max_shards: 1500
+prometheus_remote_write_enabled: false

group_vars/stage_qa/prometheus.yml

@@ -1,3 +1,2 @@
 ---
-prometheus_tsdb_rentention_time: '2w'
-kubernetes_prometheus_endpoint: "{{ stage_kube }}-prometheus.{{ domain }}"
+prometheus_remote_write_enabled: false

roles/prometheus/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 prometheus_lvm_hcloudvol_size: 30
 prometheus_lvm_hcloudvol_count: 1
-prometheus_lvm_hcloudvol_mountpath: '/prometheus_datadir'
+prometheus_lvm_hcloudvol_mountpath: "/prometheus_datadir"
 
 prometheus_datadir: "{{ prometheus_lvm_hcloudvol_mountpath }}"
 

roles/prometheus/tasks/_reload_config.yml

@@ -0,0 +1,29 @@
+---
+### tags:
+###   update_config
+
+- name: "Wait for <{{ http_s }}://{{ prometheus_id }}.{{ domain }}>"
+  uri:
+    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}"
+    url_username: "{{ prometheus_admin_username }}"
+    url_password: "{{ prometheus_admin_password }}"
+    method: GET
+    status_code: 200
+    return_content: yes
+  register: prometheus_stats
+  until: prometheus_stats.status == 200
+  retries: 10
+  delay: 60
+  tags:
+    - update_config
+
+- name: "Reload prometheus configuration"
+  uri:
+    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}/-/reload"
+    url_username: "{{ prometheus_admin_username }}"
+    url_password: "{{ prometheus_admin_password }}"
+    method: POST
+    timeout: 300 # blocks until reload configuration is complete
+    status_code: 200
+  tags:
+    - update_config

roles/prometheus/tasks/_update_config.yml

@@ -1,43 +1,32 @@
 ---
-
 ### tags:
 ###   update_config
 
-- name: "Deploy service templates for {{ inventory_hostname }}"
+- name: "Get enabled Prometheus services"
+  set_fact:
+    prometheus_services_enabled: "{{ prometheus_services_enabled | default([]) + [item] | flatten }}"
+  loop_control:
+    extended: yes
+  loop:
+    - "{{ ['alertmanager'] if prometheus_alertmanager_enabled|bool else omit }}"
+    - "{{ ['grafana'] if prometheus_grafana_enabled|bool else omit }}"
+    - "{{ ['prom2teams'] if prometheus_prom2teams_enabled|bool else omit }}"
+    - "{{ ['prometheus'] if prometheus_enabled|bool else omit }}"
+  tags:
+    - update_config
+
+- name: "Deploy Prometheus service templates for {{ inventory_hostname }}"
   include_role:
     name: hetzner-ansible-sma-deploy
     tasks_from: templates
   vars:
-    current_config: "prometheus"
+    current_config: "prometheus/{{ service }}_config"
     current_base_path: "{{ service_base_path }}"
     current_destination: "{{ inventory_hostname }}"
     current_owner: "{{ docker_owner }}"
     current_group: "{{ docker_group }}"
-  tags:
-    - update_config
-
-- name: "Wait for <{{ http_s }}://{{ prometheus_id }}.{{ domain }}>"
-  uri:
-    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}"
-    url_username: "{{ prometheus_admin_username }}"
-    url_password: "{{ prometheus_admin_password }}"
-    method: GET
-    status_code: 200
-    return_content: yes
-  register: prometheus_stats
-  until: prometheus_stats.status == 200
-  retries: 10
-  delay: 60
-  tags:
-    - update_config
-
-- name: "Reload prometheus configuration"
-  uri:
-    url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}/-/reload"
-    url_username: "{{ prometheus_admin_username }}"
-    url_password: "{{ prometheus_admin_password }}"
-    method: POST
-    timeout: 300 # blocks until reload configuration is complete
-    status_code: 200
+  loop: "{{ prometheus_services_enabled }}"
+  loop_control:
+    loop_var: service
   tags:
     - update_config

roles/prometheus/tasks/main.yml

@@ -21,10 +21,7 @@
   vars:
     record_data: "{{ stage_server_ip }}"
     record_name: "{{ item }}"
-  loop:
-    - "{{ prometheus_id }}"
-    - "{{ grafana_id }}"
-    - "{{ alertmanager_id }}"
+  loop: "{{ prometheus_service_names | reject('match', '^$') | list }}"
 
 - name: "Check if {{ inventory_hostname }}/docker-compose.yml exists"
   stat:
@@ -56,6 +53,11 @@
   tags:
     - update_deployment
 
+- name: "Updating prometheus configuration"
+  include_tasks: _update_config.yml
+  tags:
+    - update_config
+
 - name: "Update {{ inventory_hostname }}"
   community.docker.docker_compose:
     project_src: "{{ service_base_path }}/{{ inventory_hostname }}"
@@ -64,8 +66,8 @@
   tags:
     - update_deployment
 
-- name: "Updating prometheus configuration"
-  include_tasks: _update_config.yml
+- name: "Reload prometheus configuration"
+  include_tasks: _reload_config.yml
   tags:
     - update_config
 
@@ -83,6 +85,8 @@
   loop: "{{ grafana_users }}"
   tags:
     - grafana-user-update
+  when:
+    - prometheus_grafana_enabled
 
 - name: "Get all Dashboard uids from {{ http_s }}://{{ inventory_hostname }}-grafana.{{ domain }}"
   uri:
@@ -99,10 +103,12 @@
   delay: 60
   tags:
     - grafana-user-update
+  when:
+    - prometheus_grafana_enabled
 
 - name: "Get all existing Dashboard uids"
   set_fact:
-    grafana_dashboards: "{{ grafana_dashboards_plain.json | json_query('[].{uid: uid, type: type, title: title}') }}"
+    grafana_dashboards: "{{ grafana_dashboards_plain.json | json_query('[].{uid: uid, type: type, title: title}') if prometheus_grafana_enabled else [] }}"
   tags:
     - grafana-user-update
 
@@ -113,6 +119,7 @@
     - grafana-user-update
   when:
     - debug
+    - prometheus_grafana_enabled
 
 - name: Restrict admin dashboard permissions
   uri:
@@ -133,6 +140,7 @@
   when:
     - item.type == 'dash-db'
     - item.uid not in grafana_dashboard_whitelist
+    - prometheus_grafana_enabled
   tags:
     - grafana-user-update
 
@@ -155,6 +163,7 @@
   when:
     - item.type == 'dash-db'
     - item.uid in grafana_dashboard_whitelist
+    - prometheus_grafana_enabled
   tags:
     - grafana-user-update
 

roles/prometheus/vars/main.yml

@@ -1,5 +1,4 @@
 ---
-
 service_port_grafana: 3000
 service_port_prometheus: 9090
 service_port_alertmanager: 9093
@@ -9,140 +8,112 @@ alertmanager_id: "{{ inventory_hostname }}-alertmanager"
 grafana_id: "{{ inventory_hostname }}-grafana"
 prom2teams_id: "{{ inventory_hostname }}-prom2teams"
 
-prometheus_docker: {
-  networks: [
-    {
-      name: back-tier,
-      external: true,
-    },
-    {
-      name: front-tier,
-      external: true,
-    },
-  ],
-  volumes: [
-    {
-      name: "{{ alertmanager_id }}-data"
-    },
-    {
-      name: "{{ grafana_id }}-data"
-    },
-  ],
-  services: [
-    {
-      name: "{{ prometheus_id }}",
-      image_name: "{{ prometheus_image_name }}",
-      image_version: "{{ prom_prometheus_version }}",
-      user: "root",
-      labels: [
-        '"traefik.enable=true"',
-        '"traefik.http.routers.{{ prometheus_id }}.service={{ prometheus_id }}"',
-        '"traefik.http.routers.{{ prometheus_id }}.rule=Host(`{{ inventory_hostname }}-prometheus.{{ domain }}`)"',
-        '"traefik.http.routers.{{ prometheus_id }}.entrypoints=websecure"',
-        '"traefik.http.routers.{{ prometheus_id }}.tls=true"',
-        '"traefik.http.routers.{{ prometheus_id }}.tls.certresolver=letsencrypt"',
-        '"traefik.http.services.{{ prometheus_id }}.loadbalancer.server.port={{ service_port_prometheus }}"',
-        '"traefik.http.routers.{{ prometheus_id }}.middlewares={{ prometheus_id }}-basicauth"',
-        '"traefik.http.middlewares.{{ prometheus_id }}-basicauth.basicauth.users={{ prometheus_admin_username }}:{{ prometheus_admin_password_htpasswd }}"',
-      ],
-      command: [
-        '"--config.file=/etc/prometheus/prometheus.yml"',
-        '"--storage.tsdb.path=/prometheus"',
-        '"--web.console.libraries=/usr/share/prometheus/console_libraries"',
-        '"--web.console.templates=/usr/share/prometheus/consoles"',
-        '"--web.external-url={{ http_s}}://{{ inventory_hostname }}-prometheus.{{ domain }}"',
-        '"--web.enable-lifecycle"',
-        '"--storage.tsdb.retention.time={{ prometheus_tsdb_rentention_time | default("4w")}}"',
-      ],
-      volumes: [
-        '"./config/prometheus/:/etc/prometheus/:ro"',
-        '"{{ prometheus_datadir + "/" +"_data" | default(prometheus_id + "-data") }}:/prometheus"',
-      ],
-      networks: [
-        '"back-tier"',
-        '"front-tier"',
-      ],
-      extra_hosts: "{{ prometheus_extra_hosts | default([]) }}",
-    },
-    {
-      name: "{{ alertmanager_id }}",
-      image_name: "{{ alertmanager_image_name }}",
-      image_version: "{{ prom_alertmanager_version }}",
-      labels: [
-        '"traefik.enable=true"',
-        '"traefik.http.routers.{{ alertmanager_id }}.service={{ alertmanager_id }}"',
-        '"traefik.http.routers.{{ alertmanager_id }}.rule=Host(`{{ inventory_hostname }}-alertmanager.{{ domain }}`)"',
-        '"traefik.http.routers.{{ alertmanager_id }}.entrypoints=websecure"',
-        '"traefik.http.routers.{{ alertmanager_id }}.tls=true"',
-        '"traefik.http.routers.{{ alertmanager_id }}.tls.certresolver=letsencrypt"',
-        '"traefik.http.services.{{ alertmanager_id }}.loadbalancer.server.port={{ service_port_alertmanager }}"',
-        '"traefik.http.routers.{{ alertmanager_id }}.middlewares={{ alertmanager_id }}-basicauth"',
-        '"traefik.http.middlewares.{{ alertmanager_id }}-basicauth.basicauth.users={{ alertmanager_admin_username }}:{{ alertmanager_admin_password_htpasswd }}"',
-      ],
-      command: [
-        '"--config.file=/etc/alertmanager/config.yml"',
-        '"--storage.path=/alertmanager"',
-        '"--web.external-url={{ http_s}}://{{ inventory_hostname }}-alertmanager.{{ domain }}"',
-      ],
-      environment: [
-        'LS_JAVA_OPTS: "-Xmx1G -Xms1G"',
-      ],
-      volumes: [
-        '"./config/alertmanager/:/etc/alertmanager/:ro"',
-        '"{{ alertmanager_id }}-data:/alertmanager"',
-      ],
-      networks: [
-        '"back-tier"',
-        '"front-tier"',
-      ],
-    },
-    {
-      name: "{{ grafana_id }}",
-      image_name: "{{ grafana_image_name }}",
-      image_version: "{{ prom_grafana_version }}",
-      user: '"472"',
-      labels: [
-        '"traefik.enable=true"',
-        '"traefik.http.routers.{{ grafana_id }}.service={{ grafana_id }}"',
-        '"traefik.http.routers.{{ grafana_id }}.rule=Host(`{{ inventory_hostname }}-grafana.{{ domain }}`)"',
-        '"traefik.http.routers.{{ grafana_id }}.entrypoints=websecure"',
-        '"traefik.http.routers.{{ grafana_id }}.tls=true"',
-        '"traefik.http.routers.{{ grafana_id }}.tls.certresolver=letsencrypt"',
-        '"traefik.http.services.{{ grafana_id }}.loadbalancer.server.port={{ service_port_grafana }}"',
-      ],
-      volumes: [
-        '"./config/grafana/provisioning/:/etc/grafana/provisioning/"',
-        '"./config/grafana/conf/defaults.ini:/usr/share/grafana/conf/defaults.ini"',
-        '"{{ grafana_id }}-data:/var/lib/grafana"',
-      ],
-      networks: [
-        '"back-tier"',
-        '"front-tier"',
-      ],
-      env_file: [
-        '"./config/grafana/config.monitoring"',
-      ],
-    },
-    {
-      name: "{{ prom2teams_id }}",
-      image_name: "{{ prom2teams_image_name }}",
-      image_version: "{{ prom_prom2teams_version }}",
-      environment: [
-        "PROM2TEAMS_PROMETHEUS_METRICS: \"true\"",
-        "DEBUG_METRICS: \"true\"",
-      ],
-      ports: [
-        {
-          "external": "8089",
-          "internal": "8089",
-        },
-      ],
-      volumes: [
-        '"./config/prom2teams/config.ini:/opt/prom2teams/config.ini"',
-      ],
-      networks: [
-        '"back-tier"',
-      ],
-    },
-  ],
-}
+prometheus_docker:
+  networks:
+    - name: back-tier
+      external: true
+
+    - name: front-tier
+      external: true
+
+  volumes: "{{ prometheus_docker_volumes | selectattr('name', 'in', prometheus_docker_volume_names | join(',')) }}"
+
+  services: "{{ prometheus_services | selectattr('name', 'in', prometheus_service_names | join(',')) }}"
+
+prometheus_docker_volumes:
+  - name: "{{ alertmanager_id }}-data"
+  - name: "{{ grafana_id }}-data"
+
+prometheus_services:
+  - name: "{{ prometheus_id }}"
+    image_name: "{{ prometheus_image_name }}"
+    image_version: "{{ prom_prometheus_version }}"
+    user: "root"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.{{ prometheus_id }}.service={{ prometheus_id }}"
+      - "traefik.http.routers.{{ prometheus_id }}.rule=Host(`{{ inventory_hostname }}-prometheus.{{ domain }}`)"
+      - "traefik.http.routers.{{ prometheus_id }}.entrypoints=websecure"
+      - "traefik.http.routers.{{ prometheus_id }}.tls=true"
+      - "traefik.http.routers.{{ prometheus_id }}.tls.certresolver=letsencrypt"
+      - "traefik.http.services.{{ prometheus_id }}.loadbalancer.server.port={{ service_port_prometheus }}"
+      - "traefik.http.routers.{{ prometheus_id }}.middlewares={{ prometheus_id }}-basicauth"
+      - "traefik.http.middlewares.{{ prometheus_id }}-basicauth.basicauth.users={{ prometheus_admin_username }}:{{ prometheus_admin_password_htpasswd }}"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.external-url={{ http_s }}://{{ inventory_hostname }}-prometheus.{{ domain }}"
+      - "--web.enable-lifecycle"
+      - "--storage.tsdb.retention.time={{ prometheus_tsdb_rentention_time }}"
+    volumes:
+      - "./config/prometheus/:/etc/prometheus/:ro"
+      - "{{ prometheus_datadir + '/' + 'data' | default(prometheus_id + '-data') }}:/prometheus"
+    networks:
+      - back-tier
+      - front-tier
+    extra_hosts: "{{ prometheus_extra_hosts | default([]) }}"
+
+  - name: "{{ grafana_id }}"
+    image_name: "{{ grafana_image_name }}"
+    image_version: "{{ prom_grafana_version }}"
+    user: "472"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.{{ grafana_id }}.service={{ grafana_id }}"
+      - "traefik.http.routers.{{ grafana_id }}.rule=Host(`{{ inventory_hostname }}-grafana.{{ domain }}`)"
+      - "traefik.http.routers.{{ grafana_id }}.entrypoints=websecure"
+      - "traefik.http.routers.{{ grafana_id }}.tls=true"
+      - "traefik.http.routers.{{ grafana_id }}.tls.certresolver=letsencrypt"
+      - "traefik.http.services.{{ grafana_id }}.loadbalancer.server.port={{ service_port_grafana }}"
+    volumes:
+      - "./config/grafana/provisioning/:/etc/grafana/provisioning/"
+      - "./config/grafana/conf/defaults.ini:/usr/share/grafana/conf/defaults.ini"
+      - "{{ grafana_id }}-data:/var/lib/grafana"
+    networks:
+      - back-tier
+      - front-tier
+    env_file:
+      - "./config/grafana/config.monitoring"
+
+  - name: "{{ alertmanager_id }}"
+    image_name: "{{ alertmanager_image_name }}"
+    image_version: "{{ prom_alertmanager_version }}"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.{{ alertmanager_id }}.service={{ alertmanager_id }}"
+      - "traefik.http.routers.{{ alertmanager_id }}.rule=Host(`{{ inventory_hostname }}-alertmanager.{{ domain }}`)"
+      - "traefik.http.routers.{{ alertmanager_id }}.entrypoints=websecure"
+      - "traefik.http.routers.{{ alertmanager_id }}.tls=true"
+      - "traefik.http.routers.{{ alertmanager_id }}.tls.certresolver=letsencrypt"
+      - "traefik.http.services.{{ alertmanager_id }}.loadbalancer.server.port={{ service_port_alertmanager }}"
+      - "traefik.http.routers.{{ alertmanager_id }}.middlewares={{ alertmanager_id }}-basicauth"
+      - "traefik.http.middlewares.{{ alertmanager_id }}-basicauth.basicauth.users={{ alertmanager_admin_username }}:\
+        {{ alertmanager_admin_password_htpasswd }}"
+    command:
+      - "--config.file=/etc/alertmanager/config.yml"
+      - "--storage.path=/alertmanager"
+      - "--web.external-url={{ http_s }}://{{ inventory_hostname }}-alertmanager.{{ domain }}"
+    environment:
+      - 'LS_JAVA_OPTS: "-Xmx1G -Xms1G"'
+    volumes:
+      - "./config/alertmanager/:/etc/alertmanager/:ro"
+      - "{{ alertmanager_id }}-data:/alertmanager"
+    networks:
+      - back-tier
+      - front-tier
+
+  - name: "{{ prom2teams_id }}"
+    image_name: "{{ prom2teams_image_name }}"
+    image_version: "{{ prom_prom2teams_version }}"
+    environment:
+      - 'PROM2TEAMS_PROMETHEUS_METRICS: "true"'
+      - 'DEBUG_METRICS: "true"'
+    ports:
+      - external: 8089
+        internal: 8089
+    volumes:
+      - "./config/prom2teams/config.ini:/opt/prom2teams/config.ini"
+    networks:
+      - back-tier

templates/_docker/docker-compose.yml.j2

@@ -20,19 +20,21 @@ networks:
 {% endfor %}
 {% endif %}
 {# ################################################## networks #}
-
 {# ################################################### volumes #}
 {% if
   current_docker.volumes is defined
   and (current_docker.volumes|length>0)
 %}
+
 volumes:
 {% for volume in current_docker.volumes %}
   {{ volume.name }}: {}
 {% endfor %}
+
+{% else %}
+
 {% endif %}
 {# ################################################### volumes #}
-
 {# ################################################## services #}
 services:
 {% for service in current_docker.services %}
@@ -43,7 +45,7 @@ services:
 {% if
   service.user is defined
 %}
-    user: {{ service.user }}
+    user: "{{ service.user }}"
 {% endif %}
 {# ########################################## command #}
 {% if

templates/prometheus/prometheus_config/config/prometheus/prometheus.yml.j2

@@ -9,6 +9,15 @@ global:
   external_labels:
       monitor: '{{ inventory_hostname }}'
 
+{% if prometheus_remote_write_enabled | default(true) %}
+remote_write:
+  - url: "https://{{ shared_service_kube_hostname_prometheus }}/api/v1/write"
+    basic_auth:
+      username: "{{ prometheus_admin_username }}"
+      password: "{{ prometheus_admin_password }}"
+{% endif %}
+
+{% if prometheus_alertmanager_enabled | default(true) %}
 # Load and evaluate rules in this file every 'evaluation_interval' seconds.
 rule_files:
   - 'alert.rules'
@@ -20,6 +29,7 @@ alerting:
     static_configs:
     - targets:
       - "{{ inventory_hostname }}-alertmanager:9093"
+{% endif %}
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
@@ -43,6 +53,7 @@ scrape_configs:
         target_label:  instance
         replacement:   '{{ inventory_hostname }}-prometheus.{{ domain }}'
 
+{% if prometheus_alertmanager_enabled | default(true) %}
   - job_name: 'alertmanager'
     static_configs:
       - targets: [
@@ -56,7 +67,9 @@ scrape_configs:
       - source_labels: [__address__]
         target_label:  instance
         replacement:   '{{ inventory_hostname }}-alertmanager.{{ domain }}'
+{% endif %}
 
+{% if prometheus_prom2teams_enabled | default(true) %}
   - job_name: 'prom2teams'
     scheme: http
     metrics_path: '/metrics'
@@ -68,6 +81,7 @@ scrape_configs:
           env: {{ stage }}
           project: monitoring
           application: prom2teams
+{% endif %}
 
   - job_name: 'blackbox'
     metrics_path: /probe
@@ -534,6 +548,6 @@ scrape_configs:
         - '{job=~".*"}'
 
     static_configs:
-      - targets: ['{{ kubernetes_prometheus_endpoint }}']
+      - targets: ['{{ shared_service_kube_hostname_prometheus }}']
 
 {% endif %}