diff --git a/roles/logstash/tasks/main.yaml b/roles/logstash/tasks/main.yaml index bfdd009..14808a1 100644 --- a/roles/logstash/tasks/main.yaml +++ b/roles/logstash/tasks/main.yaml @@ -64,14 +64,6 @@ - update_certs - update_config -- name: "Restart {{ logstash_id }}" - community.docker.docker_compose: - project_src: '{{ service_base_path }}/{{ logstash_id }}' - restarted: yes - build: no - tags: - - update_certs - - name: "Update {{ logstash_id }}" community.docker.docker_compose: project_src: '{{ service_base_path }}/{{ logstash_id }}' diff --git a/roles/logstash/vars/main.yml b/roles/logstash/vars/main.yml index b47a750..f26aa15 100644 --- a/roles/logstash/vars/main.yml +++ b/roles/logstash/vars/main.yml @@ -14,7 +14,8 @@ elastic_docker: { image_name: "{{ logstash_image_name }}", image_version: "{{ elastic_logstash_version }}", environment: [ - "node.name: \"{{ logstash_id }}}}\"", + "log.format: \"json\"", + "node.name: \"{{ logstash_id }}\"", "config.reload.automatic: \"true\"", "pipeline.ecs_compatibility: v1", "pipeline.ordered: \"false\"", diff --git a/templates/filebeat/config/filebeat.yml.j2 b/templates/filebeat/config/filebeat.yml.j2 index 0101221..b3451a4 100644 --- a/templates/filebeat/config/filebeat.yml.j2 +++ b/templates/filebeat/config/filebeat.yml.j2 @@ -61,12 +61,18 @@ fields: stage: {{ stage }} hostname: {{ inventory_hostname }} -# DEV-650 -# avoiding container labels with [] crashing logstash +# reducing network traffic by removing unused fields +# avoiding docker.container.labels with [] crashing logstash processors: - drop_fields: fields: - - docker.container.labels + - agent + - container.id + - docker + - ecs + - host + - log + - stream ignore_missing: yes output.logstash: diff --git a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 index fa7b610..5f0c691 100644 --- a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 +++ b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 @@ -14,6 +14,8 @@ filter { if [message] =~ /^{.*}$/ { json { source => "message" + skip_on_invalid_json => true + remove_field => [ "[event][original]" ] } if [stack_trace] { ruby { @@ -48,7 +50,7 @@ filter { } } mutate { - remove_field => [ "[id]", "[agent]", "[log][file][path]", "[docker][container][labels]", "[host][ip]", "[host][mac]", "[host][name]" ] + remove_field => [ "[agent]", "[container][id]", "[docker]", "[ecs]", "[host]", "[log]", "[stream]" ] } } @@ -174,6 +176,18 @@ output { manage_template => false } } + else if [kubernetes][container][name] { + elasticsearch { + hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"] + cacert => "/usr/share/logstash/config/certificates/ca/ca.crt" + user => "{{ elastic_admin_username }}" + password => "{{ elastic_admin_password }}" + + index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][container][name]}-%{+YYYY.MM}" + + manage_template => false + } + } else { elasticsearch { hosts => ["https://{{ shared_service_elastic_stack_01_hostname }}:{{ service_port_elasticsearch }}"]