diff --git a/group_vars/stage_dev/plain.yml b/group_vars/stage_dev/plain.yml index 4b6e55c..b9af2eb 100644 --- a/group_vars/stage_dev/plain.yml +++ b/group_vars/stage_dev/plain.yml @@ -341,6 +341,9 @@ harbor_token: "{{ docker_registry_token_vault }}" elastic_admin_username: "{{ elastic_admin_username_vault }}" elastic_admin_password: "{{ elastic_admin_password_vault }}" +elastic_ilm_cold_min_age_threshold: 3d +elastic_ilm_delete_min_age_threshold: 7d + postgres_replicator_user_password: "{{ postgres_replicator_user_password_vault }}" mysql_root_username: "{{ mysql_root_username_vault }}" diff --git a/group_vars/stage_qa/plain.yml b/group_vars/stage_qa/plain.yml index b53a187..9c75e88 100644 --- a/group_vars/stage_qa/plain.yml +++ b/group_vars/stage_qa/plain.yml @@ -341,6 +341,9 @@ harbor_token: "{{ docker_registry_token_vault }}" elastic_admin_username: "{{ elastic_admin_username_vault }}" elastic_admin_password: "{{ elastic_admin_password_vault }}" +elastic_ilm_cold_min_age_threshold: 3d +elastic_ilm_delete_min_age_threshold: 7d + postgres_replicator_user_password: "{{ postgres_replicator_user_password_vault }}" mysql_root_username: "{{ mysql_root_username_vault }}" diff --git a/roles/elastic/defaults/main.yaml b/roles/elastic/defaults/main.yaml index 0337214..123e1e5 100644 --- a/roles/elastic/defaults/main.yaml +++ b/roles/elastic/defaults/main.yaml @@ -5,3 +5,53 @@ elastic_image_version: "7.16.1" elasticsearch_exporter_image_name: "quay.io/prometheuscommunity/elasticsearch-exporter" elasticsearch_exporter_image_version: "latest" + +# needs to be localhost due to "tolina" proxy aka 'banning all outgoing connection than 80/443' +elastic_api_endpoint: 'localhost:{{ service_port_elasticsearch }}' + +elastic_default_ilm_templates: + - name: "{{ stage }}-ilm-default" + template: + policy: + phases: + hot: + min_age: 0ms + actions: + rollover: + max_age: '{{ elastic_ilm_hot_max_age_threshold | default("1d") }}' + set_priority: + priority: 100 + warm: + min_age: '{{ elastic_ilm_warm_min_age_threshold | default("1d") }}' + actions: + readonly: {} + set_priority: + priority: 50 + allocate: + number_of_replicas: 1 + cold: + min_age: '{{ elastic_ilm_cold_min_age_threshold | default("7d") }}' + actions: + freeze: {} + readonly: {} + set_priority: + priority: 0 + allocate: + number_of_replicas: 1 + delete: + min_age: '{{ elastic_ilm_delete_min_age_threshold | default("30d") }}' + actions: + delete: + delete_searchable_snapshot: true + +elastic_default_index_templates: + - name: "{{ stage }}-idxtmpl-default" + template: + index_patterns: ["{{ stage }}-*"] + priority: 100 + template: + settings: + number_of_shards: 2 + index: + lifecycle: + name: "{{ stage }}-ilm-default" diff --git a/roles/elastic/tasks/create_ilm_tmpl.yml b/roles/elastic/tasks/create_ilm_tmpl.yml new file mode 100644 index 0000000..f150198 --- /dev/null +++ b/roles/elastic/tasks/create_ilm_tmpl.yml @@ -0,0 +1,59 @@ +--- +- name: "Set some VARs" + set_fact: + ilm_tmpl_state: "{{ ilm_tmpl.ilm_tmpl_state | default('present') }}" + api_object: '_ilm/policy' + tags: + - always + +- name: "Check if index template already exists" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ ilm_tmpl.name }}" + method: GET + status_code: [200,404] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + headers: + Content-Type: application/json + validate_certs: no + register: check_index_exists + tags: + - always + +- name: "Create Index template" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ ilm_tmpl.name }}" + method: PUT + status_code: [200] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + body: "{{ ilm_tmpl.template | to_json }} " + headers: + Content-Type: application/json + validate_certs: no + when: + - ilm_tmpl_state == 'present' + - check_index_exists.status in [404] + tags: + - always + +- name: "Delete Index template" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{ ilm_tmpl.name }}" + method: DELETE + status_code: [200] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + body: "{{ ilm_tmpl.template }} " + headers: + Content-Type: application/json + validate_certs: no + when: + - ilm_tmpl_state == 'absent' + - check_index_exists.status in [200] + + tags: + - always diff --git a/roles/elastic/tasks/create_index_tmpl.yml b/roles/elastic/tasks/create_index_tmpl.yml new file mode 100644 index 0000000..b619a1b --- /dev/null +++ b/roles/elastic/tasks/create_index_tmpl.yml @@ -0,0 +1,59 @@ +--- +- name: "Set some VARs" + set_fact: + idx_tmpl_state: "{{ idx_tmpl.idx_tmpl_state | default('present') }}" + api_object: '_index_template' + tags: + - always + +- name: "Check if index template already exists" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ idx_tmpl.name }}" + method: GET + status_code: [200,404] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + headers: + Content-Type: application/json + validate_certs: no + register: check_index_exists + tags: + - always + +- name: "Create Index template" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ idx_tmpl.name }}" + method: PUT + status_code: [200] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + body: "{{ idx_tmpl.template | to_json }} " + headers: + Content-Type: application/json + validate_certs: no + when: + - idx_tmpl_state == 'present' + - check_index_exists.status in [404] + tags: + - always + +- name: "Delete Index template" + uri: + url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{ idx_tmpl.name }}" + method: DELETE + status_code: [200] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + body: "{{ idx_tmpl.template }} " + headers: + Content-Type: application/json + validate_certs: no + when: + - idx_tmpl_state == 'absent' + - check_index_exists.status in [200] + + tags: + - always diff --git a/roles/elastic/tasks/main.yaml b/roles/elastic/tasks/main.yaml index 99bb24d..d99eb6c 100644 --- a/roles/elastic/tasks/main.yaml +++ b/roles/elastic/tasks/main.yaml @@ -70,6 +70,7 @@ restarted: yes build: no tags: + - never - update_certs - name: "Update {{ elastic_id }}" @@ -80,3 +81,45 @@ tags: - update_config - update_deployment + +- name: "Waiting for running elastic instance" + become: false + ansible.builtin.wait_for: + timeout: 180 + port: "{{ service_port_elasticsearch }}" + host: localhost + +- name: "Check if elastic cluster API reachable" + become: false + uri: + url: "https://{{ elastic_api_endpoint }}/_cluster/health?pretty" + method: GET + status_code: [200] + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: yes + ca_path: '{{ service_base_path }}/{{ elastic_id }}/certs/ca/ca.crt' + register: check_elastic_api_reachable + delay: 5 + retries: 15 + until: check_elastic_api_reachable.status in [200] + +- name: "Configure | create default ilm templates" + include_tasks: create_ilm_tmpl.yml + loop: '{{ elastic_default_ilm_templates }}' + loop_control: + loop_var: ilm_tmpl + tags: + - elastic-ilmtmpls + when: + - inventory_hostname == groups['elastic'][0] + +- name: "Configure | create default index templates" + include_tasks: create_index_tmpl.yml + loop: '{{ elastic_default_index_templates }}' + loop_control: + loop_var: idx_tmpl + tags: + - elastic-idxtmpls + when: + - inventory_hostname == groups['elastic'][0] diff --git a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 index 8194e23..e16e392 100644 --- a/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 +++ b/templates/logstash/config/logstash/pipeline/filebeat.conf.j2 @@ -60,7 +60,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[fields][hostname]}-authlog-%{+YYYY.MM}" + index => "%{[fields][hostname]}-authlog" + ilm_rollover_alias => "%{[fields][hostname]}-authlog" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -72,7 +74,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "uncategorized-authlog-%{+YYYY.MM}" + index => "uncategorized-authlog" + ilm_rollover_alias => "uncategorized-authlog" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -84,7 +88,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[fields][hostname]}-syslog-%{+YYYY.MM}" + index => "%{[fields][hostname]}-syslog" + ilm_rollover_alias => "%{[fields][hostname]}-syslog" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -96,7 +102,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "uncategorized-syslog-%{+YYYY.MM}" + index => "uncategorized-syslog" + ilm_rollover_alias => "uncategorized-syslog" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -108,7 +116,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[fields][hostname]}-postgresql-%{+YYYY.MM}" + index => "%{[fields][hostname]}-postgresql" + ilm_rollover_alias => "%{[fields][hostname]}-postgresql" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -120,7 +130,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[fields][hostname]}-mysql-%{+YYYY.MM}" + index => "%{[fields][hostname]}-mysql" + ilm_rollover_alias => "%{[fields][hostname]}-mysql" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -133,7 +145,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}-%{+YYYY.MM}" + index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}" + ilm_rollover_alias => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -145,7 +159,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}-%{+YYYY.MM}" + index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}" + ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -157,7 +173,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}-%{+YYYY.MM}" + index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}" + ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -169,7 +187,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}-%{+YYYY.MM}" + index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}" + ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -181,7 +201,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "uncategorized-kubernetes-%{[kubernetes][namespace]}-%{+YYYY.MM}" + index => "uncategorized-kubernetes-%{[kubernetes][namespace]}" + ilm_rollover_alias => "uncategorized-kubernetes-%{[kubernetes][namespace]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -193,7 +215,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "%{[container][name]}-%{+YYYY.MM}" + index => "%{[container][name]}" + ilm_rollover_alias => "%{[container][name]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -205,7 +229,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM}" + index => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}" + ilm_rollover_alias => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false } @@ -217,7 +243,9 @@ output { user => "{{ elastic_admin_username }}" password => "{{ elastic_admin_password }}" - index => "uncategorized-%{+YYYY.MM}" + index => "uncategorized" + ilm_rollover_alias => "uncategorized" + ilm_policy => "{{ stage }}-ilm-default" manage_template => false }