From 49fc41676495b2d1054d0e793e43ef0fc2723af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6rz=2C=20Friedrich?= Date: Thu, 7 Apr 2022 09:52:41 +0000 Subject: [PATCH] DEV-382: enable SSL for postgres-connections --- ansible.cfg | 2 +- group_vars/all/plain.yml | 1 + group_vars/stage_dev/plain.yml | 1 - group_vars/stage_prodnso/plain.yml | 1 - group_vars/stage_qa/plain.yml | 1 - roles/connect/vars/main.yml | 2 +- roles/hcloud/tasks/_set_server_state.yml | 2 +- roles/keycloak/vars/main.yml | 4 +- roles/maria/tasks/main.yml | 4 +- roles/postgres/defaults/main.yml | 126 +++++++++++++++++++++ roles/postgres/handlers/main.yml | 8 ++ roles/postgres/tasks/_postgres-acls.yml | 4 +- roles/postgres/tasks/base-requirements.yml | 92 ++++++--------- roles/restore_maria/tasks/main.yml | 2 + roles/selfsigned_ca/tasks/_create_cert.yml | 5 +- roles/webdav/vars/main.yml | 2 +- 16 files changed, 189 insertions(+), 68 deletions(-) diff --git a/ansible.cfg b/ansible.cfg index 5a9b4b7..7d7c142 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -2,7 +2,7 @@ pipelining = True host_key_checking = False inventory_plugins = ./inventory_plugins -callbacks_enabled = timer +callback_whitelist = profile_tasks interpreter_python = auto_silent log_path=last_ansible_run forks = 30 diff --git a/group_vars/all/plain.yml b/group_vars/all/plain.yml index 32fd475..60af008 100644 --- a/group_vars/all/plain.yml +++ b/group_vars/all/plain.yml @@ -209,3 +209,4 @@ k8s_basic_services: - containerd selfsigned_ca_private_key_passphrase: '{{ selfsigned_ca_private_key_passphrase_vault }}' + \ No newline at end of file diff --git a/group_vars/stage_dev/plain.yml b/group_vars/stage_dev/plain.yml index 19a4c90..2eb2dff 100644 --- a/group_vars/stage_dev/plain.yml +++ b/group_vars/stage_dev/plain.yml @@ -290,7 +290,6 @@ harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}" harbor_oidc_admin_username: "harbor-admin" harbor_oidc_admin_password: "harbor-admin" -postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'" connect_image_version: "8.5.47" iam_image_version: "latest" diff --git a/group_vars/stage_prodnso/plain.yml b/group_vars/stage_prodnso/plain.yml index f2a0f38..392a09d 100644 --- a/group_vars/stage_prodnso/plain.yml +++ b/group_vars/stage_prodnso/plain.yml @@ -290,7 +290,6 @@ harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}" harbor_oidc_admin_username: "harbor-admin" harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}" -postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'" connect_image_version: "8.5.47" iam_image_version: "latest" diff --git a/group_vars/stage_qa/plain.yml b/group_vars/stage_qa/plain.yml index 9a92769..33071a0 100644 --- a/group_vars/stage_qa/plain.yml +++ b/group_vars/stage_qa/plain.yml @@ -290,7 +290,6 @@ harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}" harbor_oidc_admin_username: "harbor-admin" harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}" -postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'" connect_image_version: "8.5.47" iam_image_version: "latest" diff --git a/roles/connect/vars/main.yml b/roles/connect/vars/main.yml index 71b3ab5..77e85e4 100644 --- a/roles/connect/vars/main.yml +++ b/roles/connect/vars/main.yml @@ -29,7 +29,7 @@ connect_environment: [ "SPRING_PROFILES_INCLUDE: \"{{ spring_profiles_include | default('swagger') }}\"", "RIBBON_DISPLAY_ON_ACTIVE_PROFILES: \"{{ ribbon_display_on_active_profiles | default('dev') }}\"", - "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_host }}:{{ service_port_postgres }}/{{ connect_postgres_database }}\"", + "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_host }}:{{ service_port_postgres }}/{{ connect_postgres_database }}?sslmode=require\"", "DATASOURCE_USERNAME: \"{{ connect_postgres_username }}\"", "DATASOURCE_PASSWORD: \"{{ connect_postgres_password }}\"", "FILE_WHITELIST_URL: \"{{ connect_whitelist_url | default('') }}\"", diff --git a/roles/hcloud/tasks/_set_server_state.yml b/roles/hcloud/tasks/_set_server_state.yml index 35ab984..5396e8e 100644 --- a/roles/hcloud/tasks/_set_server_state.yml +++ b/roles/hcloud/tasks/_set_server_state.yml @@ -37,7 +37,7 @@ - name: "DEBUG: hcloud_response" debug: msg: '{{ hcloud_response.msg }}' - + - name: "Ensure Server is STARTED when server_state=present" hetzner.hcloud.hcloud_server: api_token: "{{ hetzner_authentication_ansible }}" diff --git a/roles/keycloak/vars/main.yml b/roles/keycloak/vars/main.yml index aae7234..a199172 100644 --- a/roles/keycloak/vars/main.yml +++ b/roles/keycloak/vars/main.yml @@ -37,7 +37,7 @@ keycloak_docker: { "DB_USER: \"{{ keycloak_postgres_username }}\"", "DB_PASSWORD: \"{{ keycloak_postgres_password }}\"", "DB_ADDR: \"{{ keycloak_postgres_host }}\"", - + "JDBC_PARAMS: \"sslmode=require\"", "JAVA_OPTS_APPEND: \"-Dkeycloak.profile.feature.docker=enabled\"", ], networks: [ @@ -52,4 +52,4 @@ keycloak_docker: { extra_hosts: "{{ keycloak_extra_hosts | default([]) }}", } ], -} \ No newline at end of file +} diff --git a/roles/maria/tasks/main.yml b/roles/maria/tasks/main.yml index 8aff7c7..db3ec1f 100644 --- a/roles/maria/tasks/main.yml +++ b/roles/maria/tasks/main.yml @@ -32,10 +32,12 @@ selfsigned_ca_cert_private_key: '{{ cert_private_key }}' selfsigned_ca_cert_public_key: '{{ cert_public_key }}' selfsigned_ca_cacert: '{{ ca_cert }}' + selfsigned_ca_cert_subject: + CN: '{{ inventory_hostname }}.{{ domain }}' selfsigned_ca_cert_altnames: - 'DNS:{{ inventory_hostname }}.{{ domain }}' selfsigned_ca_trigger_handler: restart mysql - + - name: Fix binding.. ansible.builtin.lineinfile: path: /etc/mysql/mariadb.conf.d/50-server.cnf diff --git a/roles/postgres/defaults/main.yml b/roles/postgres/defaults/main.yml index 10406b3..a28bb02 100644 --- a/roles/postgres/defaults/main.yml +++ b/roles/postgres/defaults/main.yml @@ -16,8 +16,134 @@ default_shared_buffers: 256MB database_state: present +postgres_homedir: '/var/lib/postgresql' + postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_private_server_ip }}'" +postgres_base_config: + - + regex: "^data_directory" + line: "data_directory = '/var/lib/postgresql/{{ default_postgres_version }}/main'" + - + regex: "^hba_file" + line: "hba_file = '/etc/postgresql/{{ default_postgres_version }}/main/pg_hba.conf'" + - + regex: "^ident_file" + line: "ident_file = '/etc/postgresql/{{ default_postgres_version }}/main/pg_ident.conf'" + - + regex: "^external_pid_file" + line: "external_pid_file = '/var/run/postgresql/{{ default_postgres_version }}-main.pid'" + - + regex: "^port" + line: "port = 5432" + - + regex: "^unix_socket_directories" + line: "unix_socket_directories = '/var/run/postgresql'" + - + regex: "^dynamic_shared_memory_type" + line: "dynamic_shared_memory_type = posix" + - + regex: "^max_wal_size" + line: "max_wal_size = 1GB" + - + regex: "^min_wal_size" + line: "min_wal_size = 80MB" + - + regex: "^log_line_prefix" + line: "log_line_prefix = '%m [%p] %q%u@%d '" + - + regex: "^log_timezone" + line: "log_timezone = 'Etc/UTC'" + - + regex: "^cluster_name" + line: "cluster_name = '{{ default_postgres_version }}/main'" + - + regex: "^stats_temp_directory" + line: "stats_temp_directory = '/var/run/postgresql/{{ default_postgres_version }}-main.pg_stat_tmp'" + - + regex: "^datestyle" + line: "datestyle = 'iso, mdy'" + - + regex: "^timezone" + line: "timezone = 'Etc/UTC'" + - + regex: "^lc_messages" + line: "lc_messages = 'en_US.UTF-8'" + - + regex: "^lc_monetary" + line: "lc_monetary = 'en_US.UTF-8'" + - + regex: "^lc_numeric" + line: "lc_numeric = 'en_US.UTF-8'" + - + regex: "^lc_time" + line: "lc_time = 'en_US.UTF-8'" + - + regex: "^default_text_search_config" + line: "default_text_search_config = 'pg_catalog.english'" + - + regex: "^include_dir" + line: "include_dir = 'conf.d'" + - + regex: "^wal_level" + line: "wal_level = replica" + - + regex: "^max_wal_senders" + line: "max_wal_senders = 10" + - + regex: "^archive_mode" + line: "archive_mode = on" + - + regex: "^archive_command" + line: "archive_command = 'cp -f %p /postgresql/replication/%f'" + - + regex: "^wal_keep_size" + line: "wal_keep_size = 16" + - + regex: "^listen_addresses = " + line: "{{ postgres_listen_addresses }}" + - + regex: "^max_connections" + line: "max_connections = {{ default_max_connections }}" + - + regex: "^shared_buffers" + line: "shared_buffers = {{ default_shared_buffers }}" + +postgres_ssl_config: + - + regex: "^ssl = on" + line: "ssl = on" + - + regex: "^ssl_key_file" + line: "ssl_key_file = '{{ cert_private_key }}'" + - + regex: "^ssl_cert_file" + line: "ssl_cert_file = '{{ cert_public_key }}'" + - + regex: "^ssl_ca_file" + line: "ssl_ca_file = '{{ ca_cert }}'" + - + regex: "^ssl_ciphers" + line: "ssl_ciphers = 'HIGH'" + +postgres_config : '{{ postgres_base_config + ( postgres_ssl_config | default([]) ) }}' + +prometheus_postgres_exporter_config: + - + regex: "^ARGS=" + line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_postgres }}'\"" + - + regex: "^DATA_SOURCE_URI" + line: "DATA_SOURCE_URI='postgres@:5432/postgres?host=/var/run/postgresql'" + - + regex: "^PG_EXPORTER_AUTO_DISCOVER_DATABASES" + line: "PG_EXPORTER_AUTO_DISCOVER_DATABASES=true" + - + regex: "^PG_EXPORTER_EXTEND_QUERY_PATH" + line: "PG_EXPORTER_EXTEND_QUERY_PATH=/metrics/queries.yaml" + + + database_engine: postgres backup_dest_dir: "{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}" backup_status_file: '{{ backup_dest_dir }}/backup_finished' diff --git a/roles/postgres/handlers/main.yml b/roles/postgres/handlers/main.yml index cd21505..4b203d1 100644 --- a/roles/postgres/handlers/main.yml +++ b/roles/postgres/handlers/main.yml @@ -1,2 +1,10 @@ --- +- name: "restart postgres" + service: + name: postgresql + state: restarted +- name: "restart prometheus-postgres-exporter" + service: + name: prometheus-postgres-exporter + state: restarted diff --git a/roles/postgres/tasks/_postgres-acls.yml b/roles/postgres/tasks/_postgres-acls.yml index 63e506f..52ea2f8 100644 --- a/roles/postgres/tasks/_postgres-acls.yml +++ b/roles/postgres/tasks/_postgres-acls.yml @@ -9,8 +9,8 @@ - name: "Updating pg_hba.conf entries for users/nodes/schemas" lineinfile: state: '{{ database_state }}' - regex: '^host[ ]+{{ item.name }}[ ]+{{ item.name }}' - line: 'host {{ item.name }} {{ item.name }} {{ item.trusted_cidr_entry | default(shared_service_network) }} md5' + regex: '^hostssl[ ]+{{ item.name }}[ ]+{{ item.name }}' + line: 'hostssl {{ item.name }} {{ item.name }} {{ item.trusted_cidr_entry | default(shared_service_network) }} md5' path: /etc/postgresql/{{ default_postgres_version }}/main/pg_hba.conf with_items: "{{ postgres_acls }}" diff --git a/roles/postgres/tasks/base-requirements.yml b/roles/postgres/tasks/base-requirements.yml index 3464e7a..8ba9e66 100644 --- a/roles/postgres/tasks/base-requirements.yml +++ b/roles/postgres/tasks/base-requirements.yml @@ -8,7 +8,7 @@ name: postgres uid: 2000 group: postgres - home: /var/lib/postgresql + home: '{{ postgres_homedir }}' system: true shell: /bin/bash @@ -40,29 +40,40 @@ - 'postgresql-{{ default_postgres_version }}' - python3-psycopg2 -- name: "Setup listen interfaces for postgresql instance" +- name: "Set vars" + set_fact: + cert_private_key: '{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-key.pem' + cert_public_key: '{{ postgres_homedir }}/{{ inventory_hostname }}.{{ domain }}-crt.pem' + ca_cert: '{{ postgres_homedir }}/ca-certificate.pem' + +- name: "Include role for self-signed CA" + include_role: + name: selfsigned_ca + +- name: "Create certs with selfsigned CA" + include_role: + name: selfsigned_ca + tasks_from: _create_cert + vars: + selfsigned_ca_cert_private_key: '{{ cert_private_key }}' + selfsigned_ca_cert_private_key_group: postgres + selfsigned_ca_cert_public_key: '{{ cert_public_key }}' + selfsigned_ca_cacert: '{{ ca_cert }}' + selfsigned_ca_cert_subject: + CN: '{{ inventory_hostname }}.{{ domain }}' + selfsigned_ca_cert_altnames: + - 'DNS:{{ inventory_hostname }}.{{ domain }}' + - 'DNS:{{ inventory_hostname }}' +# selfsigned_ca_trigger_handler: restart postgres + +- name: "Ensure postgresql.conf via evil lineinfile..." lineinfile: state: present - regex: "^listen_addresses" - line: "{{ postgres_listen_addresses }}" + regex: "{{ item.regex }}" + line: "{{ item.line }}" path: /etc/postgresql/{{ default_postgres_version }}/main/postgresql.conf - register: listen_addresses - -- name: "Setting 'max_connections={{ default_max_connections }}'" - lineinfile: - state: present - regex: "^max_connections" - line: "max_connections = {{ default_max_connections }}" - path: /etc/postgresql/{{ default_postgres_version }}/main/postgresql.conf - register: max_connections - -- name: "Setting 'shared_buffers={{ default_shared_buffers }}'" - lineinfile: - state: present - regex: "^shared_buffers" - line: "shared_buffers = {{ default_shared_buffers }}" - path: /etc/postgresql/{{ default_postgres_version }}/main/postgresql.conf - register: shared_buffers + loop: '{{ postgres_config }}' + notify: restart postgres - name: "Creating archive directory if necessary" file: @@ -79,32 +90,14 @@ cache_valid_time: 900 state: present -- name: Setup prometheus-postgres-exporter interface bind - lineinfile: - path: /etc/default/prometheus-postgres-exporter - regex: "^ARGS=" - line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_postgres }}'\"" - -- name: "Set DATA_SOURCE_NAME in /etc/default/prometheus-postgres-exporter" +- name: "Ensure config for prometheus-postgres-exporter via evil lineinfile..." lineinfile: state: present - regex: "^DATA_SOURCE_URI" - line: "DATA_SOURCE_URI='postgres@:5432/postgres?host=/var/run/postgresql'" - path: /etc/default/prometheus-postgres-exporter - -- name: "Set PG_EXPORTER_AUTO_DISCOVER_DATABASES in /etc/default/prometheus-postgres-exporter" - lineinfile: - state: present - regex: "^PG_EXPORTER_AUTO_DISCOVER_DATABASES" - line: "PG_EXPORTER_AUTO_DISCOVER_DATABASES=true" - path: /etc/default/prometheus-postgres-exporter - -- name: "Set PG_EXPORTER_EXTEND_QUERY_PATH in /etc/default/prometheus-postgres-exporter" - lineinfile: - state: present - regex: "^PG_EXPORTER_EXTEND_QUERY_PATH" - line: "PG_EXPORTER_EXTEND_QUERY_PATH=/metrics/queries.yaml" + regex: "{{ item.regex }}" + line: "{{ item.line }}" path: /etc/default/prometheus-postgres-exporter + loop: '{{ prometheus_postgres_exporter_config }}' + notify: restart prometheus-postgres-exporter - name: "Ensure /metrics directory exists" file: @@ -142,17 +135,6 @@ file: path="/tmp/prometheus_postgres_exporter.sql" state=absent when: "role_check.stdout == '0' and server_type == 'master'" -- name: "Restarting postgres if necessary" # noqa no-handler - service: - name: postgresql - state: restarted - when: listen_addresses.changed or max_connections.changed or shared_buffers.changed - -- name: "Restart prometheus postgres exporter .." - service: - name: prometheus-postgres-exporter - state: restarted - - name: "Block: gpg stuff" become: yes become_user: postgres diff --git a/roles/restore_maria/tasks/main.yml b/roles/restore_maria/tasks/main.yml index 89bbc5c..5fd0c94 100644 --- a/roles/restore_maria/tasks/main.yml +++ b/roles/restore_maria/tasks/main.yml @@ -29,6 +29,8 @@ selfsigned_ca_cert_private_key: '{{ cert_private_key }}' selfsigned_ca_cert_public_key: '{{ cert_public_key }}' selfsigned_ca_cacert: '{{ ca_cert }}' + selfsigned_ca_cert_subject: + CN: '{{ inventory_hostname }}.{{ domain }}' selfsigned_ca_cert_altnames: - 'DNS:{{ inventory_hostname }}.{{ domain }}' selfsigned_ca_trigger_handler: restart mysql diff --git a/roles/selfsigned_ca/tasks/_create_cert.yml b/roles/selfsigned_ca/tasks/_create_cert.yml index b89c25a..60afeec 100644 --- a/roles/selfsigned_ca/tasks/_create_cert.yml +++ b/roles/selfsigned_ca/tasks/_create_cert.yml @@ -6,12 +6,13 @@ regenerate: full_idempotence size: 4096 type: RSA - group: mysql + group: '{{ selfsigned_ca_cert_private_key_group | default("root") }}' mode: '0640' - name: "Create certificate signing request (CSR) for new certificate" community.crypto.openssl_csr_pipe: privatekey_path: '{{ selfsigned_ca_cert_private_key }}' + subject: '{{ selfsigned_ca_cert_subject }}' subject_alt_name: '{{ selfsigned_ca_cert_altnames | list }}' run_once: true register: csr @@ -31,6 +32,7 @@ - name: "Write certificate file" copy: dest: '{{ selfsigned_ca_cert_public_key }}' + mode: '0644' content: "{{ certificate.certificate }}" run_once: true notify: '{{ selfsigned_ca_trigger_handler | default([]) }}' @@ -38,6 +40,7 @@ - name: "Write CA certificate" copy: src: '{{ selfsigned_ca_dir }}/ca-certificate.pem' + mode: '0644' remote_src: yes dest: '{{ selfsigned_ca_cacert }}' run_once: true diff --git a/roles/webdav/vars/main.yml b/roles/webdav/vars/main.yml index 7568230..f68807d 100644 --- a/roles/webdav/vars/main.yml +++ b/roles/webdav/vars/main.yml @@ -36,7 +36,7 @@ webdav_docker: { user: root, environment: [ "SPRING_PROFILES_INCLUDE: \"swagger,postgres\"", - "DATASOURCE_URL: \"jdbc:postgresql://{{ webdav_postgres_host }}:{{ service_port_postgres }}/{{ webdav_postgres_database }}\"", + "DATASOURCE_URL: \"jdbc:postgresql://{{ webdav_postgres_host }}:{{ service_port_postgres }}/{{ webdav_postgres_database }}?sslmode=require\"", "DATASOURCE_USERNAME: \"{{ webdav_postgres_username }}\"", "DATASOURCE_PASSWORD: \"{{ webdav_postgres_password }}\"",