DEV-416: added stuff to enable filebeat for postgres + mariabb instances

group_vars/harbor/plain.yml

@@ -2,3 +2,71 @@
 
 hetzner_server_type: cpx31
 hetzner_server_labels: "stage={{ stage }} service=harbor"
+
+filebeat_inputs:
+- type: log
+  paths:
+    - /var/log/harbor/portal.log
+  fields:
+    harbor: true
+    harbor-component: harbor-portal
+- type: log
+  paths:
+    - /var/log/harbor/exporter.log
+  fields:
+    harbor: true
+    harbor-component: harbor-exporter
+- type: log
+  paths:
+    - /var/log/harbor/redis.log
+  fields:
+    harbor: true
+    harbor-component: redis
+- type: log
+  paths:
+    - /var/log/harbor/registryctl.log
+  fields:
+    harbor: true
+    harbor-component: registryctl
+- type: log
+  paths:
+    - /var/log/harbor/chartmuseum.log
+  fields:
+    harbor: true
+    harbor-component: chartmuseum
+- type: log
+  paths:
+    - /var/log/harbor/trivy-adapter.log
+  fields:
+    harbor: true
+    harbor-component: trivy-adapter
+- type: log
+  paths:
+    - /var/log/harbor/postgresql.log
+  fields:
+    harbor: true
+    harbor-component: harbor-db
+- type: log
+  paths:
+    - /var/log/harbor/jobservice.log
+  fields:
+    harbor: true
+    harbor-component: harbor-jobservice
+- type: log
+  paths:
+    - /var/log/harbor/proxy.log
+  fields:
+    harbor: true
+    harbor-component: nginx
+- type: log
+  paths:
+    - /var/log/harbor/registry.log
+  fields:
+    harbor: true
+    harbor-component: registry
+- type: log
+  paths:
+    - /var/log/harbor/core.log
+  fields:
+    harbor: true
+    harbor-component: harbor-core

group_vars/maria/plain.yml

@@ -7,9 +7,9 @@ mysql_databases: []
 
 mysql_users: []
 
-docker_enabled: false
 traefik_enabled: false
-filebeat_enabled: false
+
+filebeat_maria_enabled: true
 
 custom_plattform_users:
   - '{{ backupuser_user_name }}'

group_vars/postgres/plain.yml

@@ -5,9 +5,9 @@ hetzner_server_labels: "stage={{ stage }} service=postgres"
 
 postgres_acls: []
 
-docker_enabled: false
 traefik_enabled: false
-filebeat_enabled: false
+
+filebeat_postgres_enabled: true
 
 custom_plattform_users:
   - '{{ backupuser_user_name }}'

roles/maria/templates/50-server.cnf

@@ -22,4 +22,7 @@ ssl                       = on
 tls_version               = TLSv1.2,TLSv1.3
 ssl_cipher                = TLSv1.2,TLSv1.3
 
+log_error                 = /var/log/mysql/error.log
+slow_query_log_file       = /var/log/mysql/slow.log
+
 [mariadb-10.6]

templates/filebeat/config/filebeat.yml.j2

@@ -1,3 +1,4 @@
+#jinja2: lstrip_blocks: "True"
 # https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
 
 filebeat.modules:
@@ -7,11 +8,42 @@ filebeat.modules:
   auth:
     enabled: true
 
+{% if filebeat_postgres_enabled | default(false) -%}
+- module: postgresql
+  log:
+    enabled: true
+{% endif -%}
+
+{% if filebeat_maria_enabled | default(false) -%}
+- module: mysql
+  error:
+    enabled: true
+  slowlog:
+    enabled: true
+{% endif -%}
+
 filebeat.inputs:
+{% if filebeat_inputs is defined %}
+{% for elem in filebeat_inputs %}
+- type: {{ elem.type }}
+  paths:
+  {% for path in elem.paths %}
+    - {{ path }}
+  {% endfor %}
+  {%+ if elem.fields is defined -%}
+  fields:
+  {% for field in elem.fields %}
+    {{ field }}: {{ elem.fields[field] }}
+  {% endfor %}
+  {% endif %}
+{% endfor %}
+{% else %}
 - type: container
   paths:
     - /var/lib/docker/containers/*/*.log
+{% endif %}
 
+{% if ansible_fqdn not in groups['harbor'] -%}
 filebeat.autodiscover:
   providers:
     - type: docker
@@ -27,6 +59,7 @@ filebeat.autodiscover:
               multiline.pattern: '^{|^[0-9]{4}-[0-9]{2}-[0-9]{2}'
               multiline.negate: true
               multiline.match: after
+{% endif %}
 
 fields:
   stage: {{ stage }}

templates/logstash/config/logstash/pipeline/filebeat.conf.j2

@@ -94,6 +94,43 @@ output {
             manage_template => false
        }
     }
+    else if [fields][hostname] and [event][dataset] == "postgresql.log" {
+        elasticsearch {
+            hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]
+            cacert => "/usr/share/logstash/config/certificates/ca/ca.crt"
+            user => "{{ elastic_admin_username }}"
+            password => "{{ elastic_admin_password }}"
+
+            index => "%{[fields][hostname]}-postgresql-%{+YYYY.MM.dd}"
+
+            manage_template => false
+       }
+    }
+    else if [fields][hostname] and (([event][dataset] == "mysql.error") or ([event][dataset] == "mysql.slowlog")){
+        elasticsearch {
+            hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]
+            cacert => "/usr/share/logstash/config/certificates/ca/ca.crt"
+            user => "{{ elastic_admin_username }}"
+            password => "{{ elastic_admin_password }}"
+
+            index => "%{[fields][hostname]}-mysql-%{+YYYY.MM.dd}"
+
+            manage_template => false
+       }
+    }
+    else if [fields][harbor] {
+
+        elasticsearch {
+            hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]
+            cacert => "/usr/share/logstash/config/certificates/ca/ca.crt"
+            user => "{{ elastic_admin_username }}"
+            password => "{{ elastic_admin_password }}"
+
+            index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}-%{+YYYY.MM.dd}"
+
+            manage_template => false
+       }
+    }
     else if [kubernetes][namespace] and [kubernetes][deployment][name] {
         elasticsearch {
             hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]