hetzner-ansible/templates/prometheus/prometheus_config/config/prometheus/prometheus.yml.j2

# my global config
global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
  # scrape_timeout is set to the global default (10s).

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
      monitor: '{{ inventory_hostname }}'

{% if prometheus_remote_write_enabled | default(true) %}
remote_write:
  - url: "https://{{ shared_service_kube_hostname_prometheus }}/api/v1/write"
    basic_auth:
      username: "{{ prometheus_admin_username }}"
      password: "{{ prometheus_admin_password }}"
{% endif %}

{% if prometheus_alertmanager_enabled | default(true) %}
# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
  - 'alert.rules'

# alert
alerting:
  alertmanagers:
  - scheme: http
    static_configs:
    - targets:
      - "{{ inventory_hostname }}-alertmanager:9093"
{% endif %}

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:

###############################################
###               Monitoring               ####
###############################################

  - job_name: 'prometheus'
    static_configs:
      - targets: [
          '{{ inventory_hostname }}-prometheus:9090'
        ]
        labels:
          env: {{ stage }}
          project: monitoring
          application: prometheus
    relabel_configs:
      - source_labels: [__address__]
        target_label:  instance
        replacement:   '{{ inventory_hostname }}-prometheus.{{ domain }}'

{% if prometheus_alertmanager_enabled | default(true) %}
  - job_name: 'alertmanager'
    static_configs:
      - targets: [
          '{{ inventory_hostname }}-alertmanager:9093'
        ]
        labels:
          env: {{ stage }}
          project: monitoring
          application: alertmanager
    relabel_configs:
      - source_labels: [__address__]
        target_label:  instance
        replacement:   '{{ inventory_hostname }}-alertmanager.{{ domain }}'
{% endif %}

{% if prometheus_prom2teams_enabled | default(true) %}
  - job_name: 'prom2teams'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
          '{{ inventory_hostname }}-prom2teams:8089'
        ]
        labels:
          env: {{ stage }}
          project: monitoring
          application: prom2teams
{% endif %}

  - job_name: 'blackbox_2xx'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets: {{ (blackbox_http_2xx_targets | default([])) + (blackbox_http_2xx_additional_targets | default([])) }}
        labels:
          env: {{ stage }}
          project: monitoring
          application: blackbox
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: "{{ shared_service_hostname_blackbox_exporter }}:{{ service_port_blackbox_exporter }}"

  - job_name: 'blackbox_3xx'
    metrics_path: /probe
    params:
      module: [http_3xx]
    static_configs:
      - targets: {{ (blackbox_http_3xx_targets | default([])) + (blackbox_http_3xx_additional_targets | default([])) }}
        labels:
          env: {{ stage }}
          project: monitoring
          application: blackbox
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: "{{ shared_service_hostname_blackbox_exporter }}:{{ service_port_blackbox_exporter }}"

  - job_name: 'blackbox_4xx'
    metrics_path: /probe
    params:
      module: [http_4xx]
    static_configs:
      - targets: {{ (blackbox_http_4xx_targets | default([])) + (blackbox_http_4xx_additional_targets | default([])) }}
        labels:
          env: {{ stage }}
          project: monitoring
          application: blackbox
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: "{{ shared_service_hostname_blackbox_exporter }}:{{ service_port_blackbox_exporter }}"

############################################
###               Traefik               ####
############################################

  - job_name: 'traefik'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if
  (hostvars[server_info.name].traefik_enabled is defined
  and hostvars[server_info.name].traefik_enabled)
  or (hostvars[server_info.name].traefik_enabled is not defined)
%}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ admin_port_service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: traefik
          application: traefik
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

###########################################
###               Harbor               ####
###########################################

  - job_name: 'harbor-exporter'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      # Scrape metrics from the Harbor exporter component
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'harbor' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ monitor_port_harbor }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-exporter
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'harbor-core'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    params:
      # Scrape metrics from the Harbor core component
      comp: ['core']
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'harbor' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ monitor_port_harbor }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-core
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'harbor-registry'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    params:
      # Scrape metrics from the Harbor registry component
      comp: ['registry']
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'harbor' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ monitor_port_harbor }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-registry
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

#############################################
###               Smardigo               ####
#############################################

  - job_name: 'connect'
    scheme: {{ http_s }}
    metrics_path: '/management/prometheus'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'connect' %}
          '{{ server_info.name }}-connect.{{ hostvars[server_info.name].domain }}:{{ admin_port_service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: connect
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'iam'
    scheme: {{ http_s }}
    metrics_path: '/management/prometheus'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'iam' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ admin_port_service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: iam
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'webdav'
    scheme: {{ http_s }}
    metrics_path: '/management/prometheus'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'webdav' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ admin_port_service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: webdav
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

#############################################
###             Smardigo Extra           ####
#############################################

  - job_name: 'connect-extra'
    scheme: {{ http_s }}
    metrics_path: '/management/prometheus'
    static_configs:
      - targets: [
{% for server in smardigo_connect_extra_servers | default([]) %}
          '{{ server }}:{{ admin_port_service }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: connect
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

#############################################
###               Keycloak               ####
#############################################

  - job_name: 'keycloak'
    scheme: {{ http_s }}
    metrics_path: '/auth/realms/master/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'keycloak' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: keycloak
          application: keycloak
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*)
        target_label:  instance
        replacement:   $1

##############################################
###                  AWX                  ####
##############################################

  - job_name: 'awx'
    scheme: {{ http_s }}
    metrics_path: '/api/v2/metrics'
    scrape_interval: 5s
    basic_auth:
      username: '{{ awx_ansible_username }}'
      password: '{{ awx_ansible_password }}'
    static_configs:
      - targets: ['{{ shared_service_kube_hostname_awx }}']
        labels:
          env: {{ stage }}
          project: awx
          application: awx
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  __address__
        replacement:   $1

###################################################
###               Infrastructure               ####
###################################################

  - job_name: 'gitea'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'gitea' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ admin_port_service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: infrastructure
          application: gitea
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

##############################################
###                Servers                ####
##############################################

  - job_name: 'node-exporter'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if
  (hostvars[server_info.name].node_exporter_enabled is defined
  and hostvars[server_info.name].node_exporter_enabled)
  or (hostvars[server_info.name].node_exporter_enabled is not defined)
%}
          '{{ server_info.private_ip }}:{{ monitor_port_system }}!{{ server_info.name }}.{{ hostvars[server_info.name].domain }}!{{ server_info.service }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: servers
    relabel_configs:
      - source_labels: [__address__]
        regex:         .*!(.*)!.*
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         .*!.*!(.*)
        target_label:  service
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*)!.*!.*
        target_label:  __address__
        replacement:   $1

  - job_name: 'node-exporter-extra'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server in node_exporter_extra_servers | default([]) %}
          '{{ server }}:{{ monitor_port_system }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: servers
    relabel_configs:
      - source_labels: [job]
        target_label:  job
        replacement:   'node-exporter'
      - source_labels: [__address__]
        regex:         .*!(.*)
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*)!.*
        target_label:  __address__
        replacement:   $1

  - job_name: 'node-exporter-external-monitoring(digitalocean)'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets:
        - '{{ shared_service_hostname_blackbox_exporter }}:{{ service_port_node_exporter }}'
        labels:
          env: {{ stage }}
          project: servers
    relabel_configs:
      - source_labels: [job]
        target_label:  job
        replacement:   'node-exporter'
      - source_labels: [__address__]
        regex:         .*!(.*)
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*)!.*
        target_label:  __address__
        replacement:   $1

#############################################
###               Database               ####
#############################################

  - job_name: 'elasticsearch-exporter'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'elastic' %}
          '{{ server_info.name }}.{{ hostvars[server_info.name].domain }}:{{ monitor_port_elastic }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: database
          application: elasticsearch
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'postgres-exporter'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'postgres' %}
          '{{ server_info.private_ip }}:{{ monitor_port_postgres }}!{{ server_info.name }}.{{ hostvars[server_info.name].domain }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: database
          application: postgres
    relabel_configs:
      - source_labels: [__address__]
        regex:         .*!(.*)
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*)!.*
        target_label:  __address__
        replacement:   $1

  - job_name: 'maria-exporter'
    scheme: http
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for server_info in stage_server_infos | default([]) %}
{% if server_info.service == 'maria' %}
          '{{ server_info.private_ip }}:{{ monitor_port_maria }}!{{ server_info.name }}.{{ hostvars[server_info.name].domain }}',
{% endif %}
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: database
          application: maria
    relabel_configs:
      - source_labels: [__address__]
        regex:         .*!(.*)
        target_label:  instance
        replacement:   $1
      - source_labels: [__address__]
        regex:         (.*)!.*
        target_label:  __address__
        replacement:   $1

{% if prometheus_federation_enabled %}
##############################################
###               Federation              ####
##############################################

  - job_name: 'federate - kube'
    scheme: https
    basic_auth:
      username: '{{ prometheus_admin_username }}'
      password: '{{ prometheus_admin_password }}'

    honor_labels: true
    metrics_path: '/federate'

    params:
      'match[]':
        - '{job=~".*"}'

    static_configs:
      - targets: ['{{ shared_service_kube_hostname_prometheus }}']

{% endif %}