DEV-579 add basic auth to prometheus stack

group_vars/all/plain.yml

@@ -126,7 +126,7 @@ default_plattform_users:
 smardigo_plattform_users: "{{ default_plattform_users + custom_plattform_users | default([]) }}"
 
 ip_whitelist_admins:
-  - "87.150.33.14/32" # sven
+  - "79.215.12.94/32" # sven
 
 ip_whitelist:
   - "212.121.131.106/32" # netgo berlin

group_vars/stage_dev/argocd.yml

@@ -1,7 +1,3 @@
 ---
 
-awx_operator_revision: "main"
 awx_smardigo_revision: "main"
-
-jaeger_operator_revision: "main"
-jaeger_smardigo_revision: "main"

group_vars/stage_dev/plain.yml

@@ -289,7 +289,7 @@ harbor_oidc_realm: "harbor"
 harbor_oidc_client_id: "harbor"
 harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}"
 harbor_oidc_admin_username: "harbor-admin"
-harbor_oidc_admin_password: "harbor-admin"
+harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}"
 
 management_oidc_realm: "management"
 management_oidc_client_id: "smardigo"
@@ -304,28 +304,27 @@ iam_jwt_enabled: true
 iam_jwt_secret: "456ae14462d049d3be76439ef379c7c6"
 
 keycloak_admin_username: "keycloak-admin"
-keycloak_admin_password: "keycloak-admin"
+keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb <username> <password>)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "$apr1$nJfFcFaI$ylS3Qa9BWAvhrMo5tWiD9."
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
-grafana_admin_password: "grafana-admin"
+grafana_admin_password: "{{ grafana_admin_password_vault }}"
 grafana_user_smardigo_login: "smardigo"
-grafana_user_smardigo_password: "smardigo"
+grafana_user_smardigo_password: "{{ grafana_user_smardigo_password_vault }}"
 grafana_signing_secret: "{{ grafana_signing_secret_vault }}"
 
 pgadmin4_admin_username: "{{ pgadmin4_admin_email }}"
-pgadmin4_admin_password: "pgadmin-admin"
+pgadmin4_admin_password: "{{ pgadmin4_admin_password_vault }}"
 
 management_admin_username: "management-admin"
-management_admin_password: "management-admin"
+management_admin_password: "{{ management_admin_password_vault }}"
 management_realm_admin_username: "management-realm-admin"
-management_realm_admin_password: "management-realm-admin"
+management_realm_admin_password: "{{ management_realm_admin_password_vault }}"
 
 harbor_admin_username: "{{ harbor_admin_username_vault }}"
 harbor_admin_password: "{{ harbor_admin_password_vault }}"
@@ -343,12 +342,12 @@ mysql_root_username: "{{ mysql_root_username_vault }}"
 mysql_root_password: "{{ mysql_root_password_vault }}"
 
 gitea_admin_username: "gitea-admin"
-gitea_admin_password: "gitea-admin"
+gitea_admin_password: "{{ gitea_admin_password_vault }}"
 gitea_realm_admin_username: "gitea-realm-admin"
 gitea_realm_admin_password: "gitea-realm-admin"
 
 argocd_admin_username: "argocd-admin"
-argocd_admin_password: "argocd-admin"
+argocd_admin_password: "{{ argocd_admin_password_vault }}"
 argo_keycloak_client_secret: "{{ argo_keycloak_client_secret_vault }}"
 argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}"
 
@@ -357,26 +356,18 @@ awx_admin_password: "{{ awx_admin_password_vault }}"
 
 prometheus_admin_username: "prometheus-admin"
 prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
-prometheus_admin_password_unencrypted: "{{ prometheus_admin_password_unencrypted_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
 
-k8s_prometheus_basic_auth_username: "prometheus-admin"
-k8s_prometheus_basic_auth_password: "{{ k8s_prometheus_basic_auth_password_vault }}"
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
 
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation DEV gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
-
-iam_opentracing_jaeger_enabled: true
-iam_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-webdav_opentracing_jaeger_enabled: true
-webdav_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-connect_opentracing_jaeger_enabled: true
-connect_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-
-prometheus_tsdb_rentention_time: '2w'

group_vars/stage_dev/prometheus.yml

@@ -0,0 +1,3 @@
+---
+
+prometheus_tsdb_rentention_time: '2w'

group_vars/stage_prodnso/plain.yml

@@ -309,9 +309,8 @@ keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb traefik-admin traefik-admin)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "{{ traefik_admin_password_vault }}"
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
 grafana_admin_password: "{{ grafana_admin_password_vault }}"
@@ -357,17 +356,18 @@ awx_admin_password: "{{ awx_admin_password_vault }}"
 
 prometheus_admin_username: "prometheus-admin"
 prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
-prometheus_admin_password_unencrypted: "{{ prometheus_admin_password_unencrypted_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
 
-k8s_prometheus_basic_auth_username: "prometheus-admin"
-k8s_prometheus_basic_auth_password: "{{ k8s_prometheus_basic_auth_password_vault }}"
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
 
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation PRODNSO gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://prodnso-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'

group_vars/stage_qa/plain.yml

@@ -309,9 +309,8 @@ keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb traefik-admin traefik-admin)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "{{ traefik_admin_password_vault }}"
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
 grafana_admin_password: "{{ grafana_admin_password_vault }}"
@@ -357,24 +356,18 @@ awx_admin_password: "{{ awx_admin_password_vault }}"
 
 prometheus_admin_username: "prometheus-admin"
 prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
-prometheus_admin_password_unencrypted: "{{ prometheus_admin_password_unencrypted_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
 
-k8s_prometheus_basic_auth_username: "prometheus-admin"
-k8s_prometheus_basic_auth_password: "{{ k8s_prometheus_basic_auth_password_vault }}"
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
 
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation QA gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://qa-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
-
-iam_opentracing_jaeger_enabled: true
-iam_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-webdav_opentracing_jaeger_enabled: true
-webdav_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-connect_opentracing_jaeger_enabled: true
-connect_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"

pip-requirements

@@ -8,3 +8,4 @@ jmespath
 netaddr
 passlib>=1.7.4
 python-jose>=3.3.0
+bcrypt==4.0.0

roles/kubernetes/argocd/tasks/main.yml

@@ -10,7 +10,6 @@
       apply:
         tags:
           - argo-cd
-    when:
     tags:
       - argo-cd
 

roles/kubernetes/prometheus/defaults/main.yml

@@ -1,15 +1,11 @@
 ---
 
-k8s_prometheus_basic_auth_secret_name: "prometheus-basic-auth" 
-k8s_prometheus_basic_auth_username: "prometheus-admin"
-k8s_prometheus_basic_auth_password: "{{ k8s_prometheus_basic_auth_password_vault }}"
-htpasswd_file_path: "/tmp/prometheus-auth"
-
-
-
 k8s_prometheus_helm__name: "prometheus"
 k8s_prometheus_helm__release_namespace: "monitoring"
 
+k8s_prometheus_basic_auth_secret_name: "prometheus-basic-auth" 
+k8s_alertmanager_basic_auth_secret_name: "alertmanager-basic-auth"
+
 # https://github.com/grafana/helm-charts
 # https://github.com/prometheus-community/helm-charts
 k8s_prometheus_helm__release_values:
@@ -47,6 +43,9 @@ k8s_prometheus_helm__release_values:
         kubernetes.io/ingress.class: nginx
         nginx.ingress.kubernetes.io/ssl-redirect: "false"
         nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ip_whitelist_admins ) | join(',') }}"
+        nginx.ingress.kubernetes.io/auth-type: "basic"
+        nginx.ingress.kubernetes.io/auth-secret: "{{ k8s_alertmanager_basic_auth_secret_name }}"
+        nginx.ingress.kubernetes.io/auth-realm: "Authentication Required"
       hosts:
         - "{{ stage }}-kube-alertmanager.{{ domain }}"
       tls:

roles/kubernetes/prometheus/tasks/_create_auth_secret.yml

@@ -0,0 +1,39 @@
+---
+
+- name: "Create empty htpswd file"
+  file:
+    path: "{{ htpasswd_file_path }}"
+    state: touch
+
+- name: "Install latest passlib with pip"
+  pip: name=passlib
+
+- name: "Add a user and password to empty htpswd file"
+  community.general.htpasswd:
+    path: "{{ htpasswd_file_path }}"
+    name: "{{ basic_auth_username }}"
+    password: "{{ basic_auth_password }}"
+
+- name: "Read credentials out of htpasswd file"
+  ansible.builtin.slurp:
+    src: "{{ htpasswd_file_path }}"
+  register: credentials
+
+- name: "Create prometheus secrets"
+  become: yes
+  kubernetes.core.k8s:
+    definition:
+      api_version: v1
+      kind: Secret
+      metadata:
+        namespace: "{{ namespace  }}"
+        name: "{{ basic_auth_secret_name }}"
+      type: Opaque
+      data: 
+        auth: "{{ credentials['content'] }}"
+
+- name: "Delete htpasswd file"
+  become: yes
+  file:
+    path: "{{ htpasswd_file_path }}"
+    state: absent

roles/kubernetes/prometheus/tasks/main.yml

@@ -3,54 +3,36 @@
 ### tags:
 ###   prometheus
 
-
-- name: Create empty htpswd file
-  file:
-    path: "{{ htpasswd_file_path }}"
-    state: touch
+- name: "Create Prometheus Basic Auth Secret"
+  include_tasks: _create_auth_secret.yml
+  vars:
+    htpasswd_file_path: "/tmp/prometheus-auth"
+    basic_auth_username: "{{ prometheus_admin_username }}"
+    basic_auth_password: "{{ prometheus_admin_password }}"
+    basic_auth_secret_name: "{{ k8s_prometheus_basic_auth_secret_name }}"
+    namespace: "{{ k8s_prometheus_helm__release_namespace }}"
+  args:
+    apply:
       tags:
         - prometheus
-
-- name: Install latest passlib with pip
-  pip: name=passlib
-
-- name: Add a user and password to empty htpswd file
-  community.general.htpasswd:
-    path: "{{ htpasswd_file_path }}"
-    name: "{{ k8s_prometheus_basic_auth_username }}"
-    password: "{{ k8s_prometheus_basic_auth_password }}"
   tags:
     - prometheus
 
-- name: read credentials out of htpasswd file
-  ansible.builtin.slurp:
-    src: "{{ htpasswd_file_path }}"
-  register: prometheus_credentials
+- name: "Create Alertmanager Basic Auth Secret"
+  include_tasks: _create_auth_secret.yml
+  vars:
+    htpasswd_file_path: "/tmp/alertmanager-auth"
+    basic_auth_username: "{{ alertmanager_admin_username }}"
+    basic_auth_password: "{{ alertmanager_admin_password }}"
+    basic_auth_secret_name: "{{ k8s_alertmanager_basic_auth_secret_name }}"
+    namespace: "{{ k8s_prometheus_helm__release_namespace }}"
+  args:
+    apply:
       tags:
         - prometheus
-
-- name: "Create prometheus secrets"
-  become: yes
-  kubernetes.core.k8s:
-    definition:
-      api_version: v1
-      kind: Secret
-      metadata:
-        namespace: "{{ k8s_prometheus_helm__release_namespace  }}"
-        name: "{{ k8s_prometheus_basic_auth_secret_name }}"
-      type: Opaque
-      data: 
-        auth: "{{ prometheus_credentials['content'] }}"
   tags:
     - prometheus
 
-- name: "delete htpasswd file"
-  become: yes
-  file:
-    path: "{{ htpasswd_file_path }}"
-    state: absent
-
-
 - name: Deploy kube-prometheus-stack inside monitoring namespace
   become: yes
   kubernetes.core.helm:

roles/prometheus/tasks/_update_config.yml

@@ -20,7 +20,7 @@
   uri:
     url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}"
     url_username: "{{ prometheus_admin_username }}"
-    url_password: "{{ prometheus_admin_password_unencrypted }}"
+    url_password: "{{ prometheus_admin_password }}"
     method: GET
     status_code: 200
     return_content: yes
@@ -35,7 +35,7 @@
   uri:
     url: "{{ http_s }}://{{ prometheus_id }}.{{ domain }}/-/reload"
     url_username: "{{ prometheus_admin_username }}"
-    url_password: "{{ prometheus_admin_password_unencrypted }}"
+    url_password: "{{ prometheus_admin_password }}"
     method: POST
     timeout: 300 # blocks until reload configuration is complete
     status_code: 200

roles/prometheus/vars/main.yml

@@ -42,7 +42,7 @@ prometheus_docker: {
         '"traefik.http.routers.{{ prometheus_id }}.tls.certresolver=letsencrypt"',
         '"traefik.http.services.{{ prometheus_id }}.loadbalancer.server.port={{ service_port_prometheus }}"',
         '"traefik.http.routers.{{ prometheus_id }}.middlewares={{ prometheus_id }}-basicauth"',
-        '"traefik.http.middlewares.{{ prometheus_id }}-basicauth.basicauth.users={{ prometheus_admin_username }}:{{ prometheus_admin_password }}"',
+        '"traefik.http.middlewares.{{ prometheus_id }}-basicauth.basicauth.users={{ prometheus_admin_username }}:{{ prometheus_admin_password_htpasswd }}"',
       ],
       command: [
         '"--config.file=/etc/prometheus/prometheus.yml"',
@@ -75,6 +75,8 @@ prometheus_docker: {
         '"traefik.http.routers.{{ alertmanager_id }}.tls=true"',
         '"traefik.http.routers.{{ alertmanager_id }}.tls.certresolver=letsencrypt"',
         '"traefik.http.services.{{ alertmanager_id }}.loadbalancer.server.port={{ service_port_alertmanager }}"',
+        '"traefik.http.routers.{{ alertmanager_id }}.middlewares={{ alertmanager_id }}-basicauth"',
+        '"traefik.http.middlewares.{{ alertmanager_id }}-basicauth.basicauth.users={{ alertmanager_admin_username }}:{{ alertmanager_admin_password_htpasswd }}"',
       ],
       command: [
         '"--config.file=/etc/alertmanager/config.yml"',

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -480,8 +480,8 @@ scrape_configs:
   - job_name: 'federate - kube'
     scheme: https
     basic_auth:
-      username: '{{ k8s_prometheus_basic_auth_username }}'
-      password: '{{ k8s_prometheus_basic_auth_password }}'
+      username: '{{ prometheus_admin_username }}'
+      password: '{{ prometheus_admin_password }}'
 
     honor_labels: true
     metrics_path: '/federate'

templates/traefik/traefik_dynamic.toml.j2

@@ -1,7 +1,7 @@
 # secure admin resources with basic authentication
 [http.middlewares.traefik-auth.basicAuth]
   users = [
-    "{{ traefik_admin_username }}:{{ traefik_admin_password }}"
+    "{{ traefik_admin_username }}:{{ traefik_admin_password_htpasswd }}"
   ]
 
 # admin api (dashboard, rest api, ...)