DEV-655 adjusted filebeat and logstash config

roles/logstash/tasks/main.yaml

@@ -64,14 +64,6 @@
     - update_certs
     - update_config
 
-- name: "Restart {{ logstash_id }}"
-  community.docker.docker_compose:
-    project_src: '{{ service_base_path }}/{{ logstash_id }}'
-    restarted: yes
-    build: no
-  tags:
-    - update_certs
-
 - name: "Update {{ logstash_id }}"
   community.docker.docker_compose:
     project_src: '{{ service_base_path }}/{{ logstash_id }}'

roles/logstash/vars/main.yml

@@ -14,7 +14,8 @@ elastic_docker: {
       image_name: "{{ logstash_image_name }}",
       image_version: "{{ elastic_logstash_version }}",
       environment: [
-        "node.name: \"{{ logstash_id }}}}\"",
+        "log.format: \"json\"",
+        "node.name: \"{{ logstash_id }}\"",
         "config.reload.automatic: \"true\"",
         "pipeline.ecs_compatibility: v1",
         "pipeline.ordered: \"false\"",

templates/filebeat/config/filebeat.yml.j2

@@ -61,12 +61,18 @@ fields:
   stage: {{ stage }}
   hostname: {{ inventory_hostname }}
 
-# DEV-650
-# avoiding container labels with [] crashing logstash
+# reducing network traffic by removing unused fields 
+# avoiding docker.container.labels with [] crashing logstash
 processors:
   - drop_fields:
       fields:
-        - docker.container.labels
+        - agent
+        - container.id
+        - docker
+        - ecs
+        - host
+        - log
+        - stream
       ignore_missing: yes
 
 output.logstash:

templates/logstash/config/logstash/pipeline/filebeat.conf.j2

@@ -14,6 +14,8 @@ filter {
     if [message] =~ /^{.*}$/ {
         json {
             source => "message"
+            skip_on_invalid_json => true
+            remove_field => [ "[event][original]" ]
         }
         if [stack_trace] {
             ruby {
@@ -48,7 +50,7 @@ filter {
         }
     }
     mutate {
-      remove_field => [ "[id]", "[agent]", "[log][file][path]", "[docker][container][labels]", "[host][ip]", "[host][mac]", "[host][name]" ]
+      remove_field => [ "[agent]", "[container][id]", "[docker]", "[ecs]", "[host]", "[log]", "[stream]" ]
     }
 }
 
@@ -174,6 +176,18 @@ output {
                 manage_template => false
            }
         }
+        else if [kubernetes][container][name] {
+            elasticsearch {
+                hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]
+                cacert => "/usr/share/logstash/config/certificates/ca/ca.crt"
+                user => "{{ elastic_admin_username }}"
+                password => "{{ elastic_admin_password }}"
+
+                index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][container][name]}-%{+YYYY.MM}"
+
+                manage_template => false
+           }
+        }
         else {
             elasticsearch {
                 hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]