DEV-416: review collect postgres logs to elk-stack

templates/filebeat/config/filebeat.yml.j2

@@ -7,43 +7,39 @@ filebeat.modules:
     enabled: true
   auth:
     enabled: true
+{% if filebeat_postgres_enabled | default(false) %}
 
-{% if filebeat_postgres_enabled | default(false) -%}
 - module: postgresql
   log:
     enabled: true
-{% endif -%}
+{% endif %}
+{% if filebeat_maria_enabled | default(false) %}
 
-{% if filebeat_maria_enabled | default(false) -%}
 - module: mysql
   error:
     enabled: true
   slowlog:
     enabled: true
 {% endif -%}
+{% if filebeat_inputs is defined %}
 
 filebeat.inputs:
-{% if filebeat_inputs is defined %}
+{% for filebeat_input in filebeat_inputs %}
-{% for elem in filebeat_inputs %}
+- type: {{ filebeat_input.type }}
-- type: {{ elem.type }}
   paths:
-  {% for path in elem.paths %}
+  {% for path in filebeat_input.paths %}
     - {{ path }}
   {% endfor %}
-  {%+ if elem.fields is defined -%}
+  {%+ if filebeat_input.fields is defined -%}
   fields:
-  {% for field in elem.fields %}
+  {% for field in filebeat_input.fields %}
-    {{ field }}: {{ elem.fields[field] }}
+    {{ field }}: {{ filebeat_input.fields[field] }}
   {% endfor %}
   {% endif %}
 {% endfor %}
-{% else %}
-- type: container
-  paths:
-    - /var/lib/docker/containers/*/*.log
 {% endif %}
+{% if ansible_fqdn not in groups['harbor'] %}
 
-{% if ansible_fqdn not in groups['harbor'] -%}
 filebeat.autodiscover:
   providers:
     - type: docker

templates/logstash/config/logstash/pipeline/filebeat.conf.j2

@@ -37,11 +37,18 @@ filter {
             pattern_definitions => {
                 "GREEDYMULTILINE"=> "(.|\n)*"
             }
-            remove_field => "message"
+            remove_field => [ "message", "[event][original]" ]
+        }
+    }
+    else if [event][dataset] == "postgresql.log" {
+        grok {
+            match => {
+                     "message" => "%{DATESTAMP:timestamp} %{TZ} (\[%{DATA:group_id}\]) (\[?%{DATA:user}\]?@\[?%{DATA:database}\]? )?%{DATA:level}:  %{GREEDYDATA:message_full}" }
+            remove_field => [ "message", "[event][original]" ]
         }
     }
     mutate {
-      remove_field => [ "[id]", "[agent]", "[log][file][path]", "[docker][container][labels]", "[host][ip]", "[host][mac]" ]
+      remove_field => [ "[id]", "[agent]", "[log][file][path]", "[docker][container][labels]", "[host][ip]", "[host][mac]", "[host][name]" ]
     }
 }