diff --git a/group_vars/harbor/plain.yml b/group_vars/harbor/plain.yml index b9e7b7b..5f503d9 100644 --- a/group_vars/harbor/plain.yml +++ b/group_vars/harbor/plain.yml @@ -2,3 +2,71 @@ hetzner_server_type: cpx31 hetzner_server_labels: "stage={{ stage }} service=harbor" + +filebeat_inputs: +- type: log + paths: + - /var/log/harbor/portal.log + fields: + harbor: true + harbor-component: harbor-portal +- type: log + paths: + - /var/log/harbor/exporter.log + fields: + harbor: true + harbor-component: harbor-exporter +- type: log + paths: + - /var/log/harbor/redis.log + fields: + harbor: true + harbor-component: redis +- type: log + paths: + - /var/log/harbor/registryctl.log + fields: + harbor: true + harbor-component: registryctl +- type: log + paths: + - /var/log/harbor/chartmuseum.log + fields: + harbor: true + harbor-component: chartmuseum +- type: log + paths: + - /var/log/harbor/trivy-adapter.log + fields: + harbor: true + harbor-component: trivy-adapter +- type: log + paths: + - /var/log/harbor/postgresql.log + fields: + harbor: true + harbor-component: harbor-db +- type: log + paths: + - /var/log/harbor/jobservice.log + fields: + harbor: true + harbor-component: harbor-jobservice +- type: log + paths: + - /var/log/harbor/proxy.log + fields: + harbor: true + harbor-component: nginx +- type: log + paths: + - /var/log/harbor/registry.log + fields: + harbor: true + harbor-component: registry +- type: log + paths: + - /var/log/harbor/core.log + fields: + harbor: true + harbor-component: harbor-core diff --git a/group_vars/maria/plain.yml b/group_vars/maria/plain.yml index 1c5a543..ec9f536 100644 --- a/group_vars/maria/plain.yml +++ b/group_vars/maria/plain.yml @@ -7,9 +7,9 @@ mysql_databases: [] mysql_users: [] -docker_enabled: false traefik_enabled: false -filebeat_enabled: false + +filebeat_maria_enabled: true custom_plattform_users: - '{{ backupuser_user_name }}' diff --git a/group_vars/postgres/plain.yml b/group_vars/postgres/plain.yml index d804c1e..b91d393 100644 --- a/group_vars/postgres/plain.yml +++ b/group_vars/postgres/plain.yml @@ -5,9 +5,9 @@ hetzner_server_labels: "stage={{ stage }} service=postgres" postgres_acls: [] -docker_enabled: false traefik_enabled: false -filebeat_enabled: false + +filebeat_postgres_enabled: true custom_plattform_users: - '{{ backupuser_user_name }}' diff --git a/roles/maria/templates/50-server.cnf b/roles/maria/templates/50-server.cnf index 6bfeacd..2d7dcfb 100644 --- a/roles/maria/templates/50-server.cnf +++ b/roles/maria/templates/50-server.cnf @@ -22,4 +22,7 @@ ssl = on tls_version = TLSv1.2,TLSv1.3 ssl_cipher = TLSv1.2,TLSv1.3 +log_error = /var/log/mysql/error.log +slow_query_log_file = /var/log/mysql/slow.log + [mariadb-10.6] diff --git a/templates/filebeat/config/filebeat.yml.j2 b/templates/filebeat/config/filebeat.yml.j2 index 338cc5a..13216bd 100644 --- a/templates/filebeat/config/filebeat.yml.j2 +++ b/templates/filebeat/config/filebeat.yml.j2 @@ -1,3 +1,4 @@ +#jinja2: lstrip_blocks: "True" # https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html filebeat.modules: @@ -7,11 +8,42 @@ filebeat.modules: auth: enabled: true +{% if filebeat_postgres_enabled | default(false) -%} +- module: postgresql + log: + enabled: true +{% endif -%} + +{% if filebeat_maria_enabled | default(false) -%} +- module: mysql + error: + enabled: true + slowlog: + enabled: true +{% endif -%} + filebeat.inputs: +{% if filebeat_inputs is defined %} +{% for elem in filebeat_inputs %} +- type: {{ elem.type }} + paths: + {% for path in elem.paths %} + - {{ path }} + {% endfor %} + {%+ if elem.fields is defined -%} + fields: + {% for field in elem.fields %} + {{ field }}: {{ elem.fields[field] }} + {% endfor %} + {% endif %} +{% endfor %} +{% else %} - type: container paths: - -/var/lib/docker/containers/*/*.log + - /var/lib/docker/containers/*/*.log +{% endif %} +{% if ansible_fqdn not in groups['harbor'] -%} filebeat.autodiscover: providers: - type: docker @@ -27,6 +59,7 @@ filebeat.autodiscover: multiline.pattern: '^{|^[0-9]{4}-[0-9]{2}-[0-9]{2}' multiline.negate: true multiline.match: after +{% endif %} fields: stage: {{ stage }} diff --git a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 index 1df6c40..1049e0c 100644 --- a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 +++ b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 @@ -94,6 +94,43 @@ output { manage_template => false } } + else if [fields][hostname] and [event][dataset] == "postgresql.log" { + elasticsearch { + hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"] + cacert => "/usr/share/logstash/config/certificates/ca/ca.crt" + user => "{{ elastic_admin_username }}" + password => "{{ elastic_admin_password }}" + + index => "%{[fields][hostname]}-postgresql-%{+YYYY.MM.dd}" + + manage_template => false + } + } + else if [fields][hostname] and (([event][dataset] == "mysql.error") or ([event][dataset] == "mysql.slowlog")){ + elasticsearch { + hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"] + cacert => "/usr/share/logstash/config/certificates/ca/ca.crt" + user => "{{ elastic_admin_username }}" + password => "{{ elastic_admin_password }}" + + index => "%{[fields][hostname]}-mysql-%{+YYYY.MM.dd}" + + manage_template => false + } + } + else if [fields][harbor] { + + elasticsearch { + hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"] + cacert => "/usr/share/logstash/config/certificates/ca/ca.crt" + user => "{{ elastic_admin_username }}" + password => "{{ elastic_admin_password }}" + + index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}-%{+YYYY.MM.dd}" + + manage_template => false + } + } else if [kubernetes][namespace] and [kubernetes][deployment][name] { elasticsearch { hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]