hetzner-ansible/templates/prometheus/config/prometheus/prometheus.yml.j2

# my global config
global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
  # scrape_timeout is set to the global default (10s).

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
      monitor: '{{ stage_server_name }}'

# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
  - 'alert.rules'

# alert
alerting:
  alertmanagers:
  - scheme: http
    static_configs:
    - targets:
      - "{{ service_name }}-alertmanager:9093"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:

##############################################
###              Apllications             ####
##############################################

  - job_name: 'prometheus'
    static_configs:
      - targets: [
          '{{ service_name }}-prometheus:9090'
        ]
        labels:
          env: {{ stage }}
          project: smardigo
    relabel_configs:
      - source_labels: [__address__]
        target_label:  instance
        replacement:   'prometheus.{{ domain }}'

##############################################

  - job_name: 'traefik'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for host in groups['all'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_service }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: traefik
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

##############################################

  - job_name: 'harbor-exporter'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      # Scrape metrics from the Harbor exporter component
      - targets: [
{% for host in groups['harbor'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_harbor }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-exporter
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'harbor-core'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    params:
      # Scrape metrics from the Harbor core component
      comp: ['core']
    static_configs:
      - targets: [
{% for host in groups['harbor'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_harbor }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-core
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

  - job_name: 'harbor-registry'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    params:
      # Scrape metrics from the Harbor registry component
      comp: ['registry']
    static_configs:
      - targets: [
{% for host in groups['harbor'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_harbor }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: harbor
          application: harbor-registry
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

##############################################

  - job_name: 'connect'
    scheme: {{ http_s }}
    metrics_path: '/management/prometheus'
    static_configs:
      - targets: [
{% for host in groups['connect'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_service }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
          application: connect
    relabel_configs:
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

##############################################
###                Servers                ####
##############################################

  - job_name: 'node-exporter'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for host in groups['stage_dev'] | default([]) %}
          '{{ host }}.{{ domain }}:{{ monitor_port_system }}',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
    relabel_configs:
      - source_labels: [job]
        target_label:  job
        replacement:   'node-exporter'
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1

##############################################
###               Databases               ####
##############################################

  - job_name: 'elasticsearch-exporter'
    scheme: {{ http_s }}
    metrics_path: '/metrics'
    static_configs:
      - targets: [
{% for host in groups['elastic'] | default([]) %}
          '{{ host }}.{{ domain }}:9083',
{% endfor %}
        ]
        labels:
          env: {{ stage }}
          project: smardigo
    relabel_configs:
      - source_labels: [job]
        target_label:  job
        replacement:   'elasticsearch-exporter'
      - source_labels: [__address__]
        regex:         (.*):.*
        target_label:  instance
        replacement:   $1