DEV-429: mariadb upgrade

group_vars/all/plain.yml

@@ -43,6 +43,7 @@ common_apt_dependencies:
   - net-tools
   - bash-completion
   - python3-pip
+  - iotop
 
 common_pip_dependencies:
   - docker-compose

restore-remote-database-backup.yml

@@ -197,9 +197,7 @@
         become: yes
         become_user: root
         community.mysql.mysql_query:
-          login_user: '{{ mysql_root_username }}'
-          login_password: "{{ mysql_root_password }}"
-          config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+          login_unix_socket: /run/mysqld/mysqld.sock
           login_db: dummytestdb
           query: SELECT movie FROM movie_quotes WHERE quote = %s
           positional_args:

roles/backup/files/push_backups_to_restore_server.sh

@@ -12,7 +12,7 @@ DATABASE_ENGINE=$3
 DATE=$(date +%F)
 
 LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}"
-BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | head -n 1)
+BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1)
 
 REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}"
 DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"

roles/connect_wordpress/vars/main.yml

@@ -42,7 +42,7 @@ wordpress_docker: {
         "WORDPRESS_CONFIG_EXTRA: |",
         "  define( 'WP_HOME', 'https://{{ wordpress_base_url }}' );",
         "  define( 'WP_SITEURL', 'https://{{ wordpress_base_url }}' );",
-#        "  define( 'MYSQL_CLIENT_FLAGS', MYSQLI_CLIENT_SSL | MYSQLI_CLIENT_SSL_DONT_VERIFY_SERVER_CERT );",
+        "  define( 'MYSQL_CLIENT_FLAGS', MYSQLI_CLIENT_SSL | MYSQLI_CLIENT_SSL_DONT_VERIFY_SERVER_CERT );",
         "AUTH_API: \"https://{{ shared_service_keycloak_hostname }}\"",
         "RESOURCE_API: \"https://{{ connect_base_url }}\"",
         "REALM_ID: \"{{ current_realm_name }}\"",

roles/import_maria_database/tasks/main.yml

@@ -6,7 +6,7 @@
   community.mysql.mysql_db:
     name: "{{ target_database }}"
     state: absent
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /var/run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
 
 - name: "Import database from <{{ upload_directory }}/{{ database_backup_file }}> to <{{ target_database }}>"
@@ -14,5 +14,5 @@
     name: "{{ target_database }}"
     state: import
     target: "/{{ upload_directory }}/{{ database_backup_file }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /var/run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"

roles/maria/defaults/main.yml

@@ -1,5 +1,14 @@
 ---
+ansible_managed: 'do not edit manually - file powered by ansible'
+
+mariadb_server_version: '10.6'
+mariadb_server_global_my_cnf: '/etc/mysql/my.cnf'
+mariadb_server_config_dir: '/etc/mysql/mariadb.conf.d'
+
 my_cnf_file: '/root/.my.cnf'
 database_engine: maria
 backup_dest_dir: "{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}"
 backup_status_file: '{{ backup_dest_dir }}/backup_finished'
+
+mysql_root_username: "{{ mysql_root_username_vault }}"
+mysql_root_password: "{{ mysql_root_password_vault }}"

roles/maria/tasks/_create_database.yml

@@ -6,7 +6,7 @@
     collation: "{{ item.collation | default('utf8_general_ci') }}"
     encoding: "{{ item.encoding | default('utf8') }}"
     state: "{{ item.state | default('present') }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /var/run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
   with_items: "{{ mysql_databases }}"
 
@@ -18,7 +18,7 @@
     state: "{{ item.state | default('present') }}"
     append_privs: "{{ item.append_privs | default('no') }}"
     encrypted: "{{ item.encrypted | default('no') }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /var/run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
     host: "{{ item.host }}"
   with_items: "{{ mysql_users }}"

roles/maria/tasks/install_mysqld_exporter.yml

@@ -0,0 +1,40 @@
+---
+# task bundle simply copied from main.yml
+# TODO: migrate to https://github.com/cloudalchemy/ansible-mysqld_exporter
+- name: "Install prometheus-mysqld-exporter"# noqa package-latest
+  package:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - prometheus-mysqld-exporter
+
+- name: Ensure prometheus user for prometheus-mysqld-exporter exists
+  community.mysql.mysql_user:
+    name: "prometheus"
+    priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT"
+    login_unix_socket: /run/mysqld/mysqld.sock
+    login_password: "{{ mysql_root_password }}"
+  register: mysql_exporter_user_creds
+  notify: prometheus-mysqld-exporter restart
+
+- name: Ensure is prometheus-mysqld-exporter configured
+  lineinfile:
+    regex: "^DATA_SOURCE_NAME="
+    line: 'DATA_SOURCE_NAME="prometheus@unix(/run/mysqld/mysqld.sock)/"'
+    path: /etc/default/prometheus-mysqld-exporter
+  register: mysql_exporter_data_source
+  notify: prometheus-mysqld-exporter restart
+
+- name: Setup prometheus-mysqld-exporter interface bind
+  lineinfile:
+    path: /etc/default/prometheus-mysqld-exporter
+    regex: "^ARGS="
+    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_maria }}'\""
+  register: mysql_exporter_args
+  notify: prometheus-mysqld-exporter restart
+
+- name: "Ensure prometheus-mysqld-exporter is running"
+  service:
+    name: prometheus-mysqld-exporter
+    state: started
+    enabled: yes

roles/maria/tasks/main.yml

@@ -2,10 +2,17 @@
 
 ### tags:
 
-- name: Update
-  apt: update_cache=yes force_apt_get=yes cache_valid_time=3600
+- name: "Add apt-key for "
+  ansible.builtin.apt_key:
+    url: https://mariadb.org/mariadb_release_signing_key.asc
+    state: present
+
+- name: "Add source repository for mariadb-server"
+  ansible.builtin.apt_repository:
+    repo: "deb [arch=amd64] https://ftp.agdsn.de/pub/mirrors/mariadb/repo/{{ mariadb_server_version }}/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} main"
+    state: present
 
-- name: MariaDB | install # noqa package-latest
+- name: "Install MariaDB "# noqa package-latest
   package:
     name: "{{ item }}"
     state: latest
@@ -13,13 +20,12 @@
     - mariadb-server
     - mariadb-backup
     - python3-pymysql
-    - prometheus-mysqld-exporter
 
 - name: "Set vars"
   set_fact:
-    cert_private_key: '/etc/mysql/conf.d/{{ inventory_hostname }}.{{ domain }}-key.pem'
-    cert_public_key: '/etc/mysql/conf.d/{{ inventory_hostname }}.{{ domain }}-crt.pem'
-    ca_cert: '/etc/mysql/conf.d/ca-certificate.pem'
+    cert_private_key: '{{ mariadb_server_config_dir }}/{{ inventory_hostname }}.{{ domain }}-key.pem'
+    cert_public_key: '{{ mariadb_server_config_dir }}/{{ inventory_hostname }}.{{ domain }}-crt.pem'
+    ca_cert: '{{ mariadb_server_config_dir }}/ca-certificate.pem'
 
 - name: "Include role for self-signed CA"
   include_role:
@@ -41,28 +47,26 @@
       - 'DNS:{{ inventory_hostname }}'
     selfsigned_ca_trigger_handler: restart mysql
 
-- name: Fix binding..
-  ansible.builtin.lineinfile:
-    path: /etc/mysql/mariadb.conf.d/50-server.cnf
-    regexp: '^bind-address'
-    line: 'bind-address={{ stage_private_server_ip }}'
+- name: "Create global my.cnf for mariadb"
+  copy:
+    dest: '{{ mariadb_server_global_my_cnf }}'
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      {{ ansible_managed | comment }}
+      !includedir /etc/mysql/mariadb.conf.d/
   notify: restart mysql
 
-# DEV-422: SSL stuff does not work as expected
-#- name: "Create my.cnf containing ssl stuff"
-#  template:
-#    src: 50-ssl.cnf
-#    dest: /etc/mysql/conf.d/
-#    mode: '0644'
-#    owner: root
-#    group: root
-#  notify: restart mysql
-
-# DEV-422
-- name: "Ensure configured SSL config is removed"
-  file:
-    state: absent
-    path: /etc/mysql/conf.d/50-ssl.cnf
+- name: "Create mariadb cnf file"
+  vars:
+    mariadb_server_bind_address: '{{ stage_private_server_ip }}'
+  template:
+    src: 50-server.cnf
+    dest: '{{ mariadb_server_config_dir }}/'
+    mode: '0644'
+    owner: root
+    group: root
   notify: restart mysql
 
 - name: Ensure service is started
@@ -93,9 +97,10 @@
     collation: "{{ item.collation | default('utf8_general_ci') }}"
     encoding: "{{ item.encoding | default('utf8') }}"
     state: "{{ item.state | default('present') }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
   with_items: "{{ mysql_databases }}"
+  when: mysql_databases is defined
 
 - name: Ensure MySQL users are present.
   community.mysql.mysql_user:
@@ -105,41 +110,15 @@
     state: "{{ item.state | default('present') }}"
     append_privs: "{{ item.append_privs | default('no') }}"
     encrypted: "{{ item.encrypted | default('no') }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
+    login_unix_socket: /run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
     host: "{{ item.host }}"
   with_items: "{{ mysql_users }}"
+  when: mysql_users is defined
 
-- name: Ensure prometheus user for prometheus-mysqld-exporter exists
-  community.mysql.mysql_user:
-    name: "prometheus"
-    priv: "*.*:PROCESS,REPLICATION CLIENT,SELECT"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
-    login_password: "{{ mysql_root_password }}"
-  register: mysql_exporter_user_creds
-  notify: prometheus-mysqld-exporter restart
-
-- name: Ensure is prometheus-mysqld-exporter configured
-  lineinfile:
-    regex: "^DATA_SOURCE_NAME="
-    line: 'DATA_SOURCE_NAME="prometheus@unix(/run/mysqld/mysqld.sock)/"'
-    path: /etc/default/prometheus-mysqld-exporter
-  register: mysql_exporter_data_source
-  notify: prometheus-mysqld-exporter restart
-
-- name: Setup prometheus-mysqld-exporter interface bind
-  lineinfile:
-    path: /etc/default/prometheus-mysqld-exporter
-    regex: "^ARGS="
-    line: "ARGS=\"--web.listen-address='{{ stage_private_server_ip }}:{{ monitor_port_maria }}'\""
-  register: mysql_exporter_args
-  notify: prometheus-mysqld-exporter restart
-
-- name: "Ensure prometheus-mysqld-exporter is running"
-  service:
-    name: prometheus-mysqld-exporter
-    state: started
-    enabled: yes
+- name: "Install promethues mysqld-exporter"
+  include_tasks: install_mysqld_exporter.yml
+  when: mariadb_server_with_mysqld_exporter | default(True)
 
 - name: 'Ensures <{{ backup_directory }}> directory exists'
   file:
@@ -159,9 +138,8 @@
 
 - name: "Ensure test DB"
   community.mysql.mysql_db:
-    login_user: '{{ mysql_root_username }}'
+    login_unix_socket: /run/mysqld/mysqld.sock
     login_password: "{{ mysql_root_password }}"
-    config_file: "/etc/mysql/mariadb.conf.d/50-client.cnf"
     name: dummytestdb
     state: import
     target: /tmp/testdb.sql

roles/maria/templates/50-server.cnf

@@ -0,0 +1,25 @@
+{{ ansible_managed | comment }}
+
+[server]
+
+[mysqld]
+
+pid-file                  = /run/mysqld/mysqld.pid
+basedir                   = /usr
+bind-address              = {{ mariadb_server_bind_address }}
+expire_logs_days          = 10
+character-set-server      = utf8mb4
+collation-server          = utf8mb4_general_ci
+
+[embedded]
+
+[mariadb]
+require_secure_transport = on
+ssl_key                   = {{ cert_private_key }}
+ssl_cert                  = {{ cert_public_key }}
+ssl_ca                    = {{ ca_cert }}
+ssl                       = on
+tls_version               = TLSv1.2,TLSv1.3
+ssl_cipher                = TLSv1.2,TLSv1.3
+
+[mariadb-10.6]

roles/maria/templates/50-ssl.cnf

@@ -1,7 +0,0 @@
-[mysqld]
-ssl_key = {{ cert_private_key }}
-ssl_cert = {{ cert_public_key }} 
-ssl_ca = {{ ca_cert }}
-ssl = on
-tls_version = TLSv1.2,TLSv1.3
-ssl_cipher = TLSv1.2,TLSv1.3

roles/restore_maria/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+mariadb_server_with_mysqld_exporter: False

roles/restore_maria/tasks/main.yml

@@ -1,10 +1,4 @@
 ---
-- name: "Set vars"
-  set_fact:
-    cert_private_key: '/etc/mysql/conf.d/{{ inventory_hostname }}.{{ domain }}-key.pem'
-    cert_public_key: '/etc/mysql/conf.d/{{ inventory_hostname }}.{{ domain }}-crt.pem'
-    ca_cert: '/etc/mysql/conf.d/ca-certificate.pem'
-
 # DEV-375
 # "fixes" error for mysql-connect as root-user
 # it's just a restore server ...
@@ -19,36 +13,9 @@
       user={{ mysql_root_username }}
       password={{ mysql_root_password }}
 
-- name: "Install mariadb via include_role"
-  vars:
-    mysql_packages:
-      - mariadb-client
-      - mariadb-server
-      - mariadb-backup
-    mysql_bind_address: '{{ stage_private_server_ip }}'
-#    mysql_config_include_files:
-#      - src: 50-ssl.cnf
-  include_role:
-    name: geerlingguy.mysql
-
-- name: "Include role for self-signed CA"
-  include_role:
-    name: selfsigned_ca
-
-- name: "Create certs with selfsigned CA"
+- name: "Install mariadb-server via include_role"
   include_role:
-    name: selfsigned_ca
-    tasks_from: _create_cert
-  vars:
-    selfsigned_ca_cert_private_key: '{{ cert_private_key }}'
-    selfsigned_ca_cert_private_key_group: mysql
-    selfsigned_ca_cert_public_key: '{{ cert_public_key }}'
-    selfsigned_ca_cacert: '{{ ca_cert }}'
-    selfsigned_ca_cert_subject:
-      CN: '{{ inventory_hostname }}.{{ domain }}'
-    selfsigned_ca_cert_altnames:
-      - 'DNS:{{ inventory_hostname }}.{{ domain }}'
-    selfsigned_ca_trigger_handler: restart mysql
+    name: maria
 
 - name: "Copy restore script to restore server"
   copy:

roles/selfsigned_ca/tasks/_create_cert.yml

@@ -1,4 +1,12 @@
 ---
+- name: "Ensure directory"
+  file:
+    path: '{{ selfsigned_ca_cert_private_key | dirname }}'
+    state: directory
+    mode: '0755'
+    owner: root
+    group: root
+
 - name: "Generate an OpenSSL private key"
   community.crypto.openssl_privatekey:
     path: '{{ selfsigned_ca_cert_private_key }}'