DEV-470: debugging, fixing , crying

group_vars/stage_dev/plain.yml

@@ -341,6 +341,9 @@ harbor_token: "{{ docker_registry_token_vault }}"
 elastic_admin_username: "{{ elastic_admin_username_vault }}"
 elastic_admin_password: "{{ elastic_admin_password_vault }}"
 
+elastic_ilm_cold_min_age_threshold: 3d
+elastic_ilm_delete_min_age_threshold: 7d
+
 postgres_replicator_user_password: "{{ postgres_replicator_user_password_vault }}"
 
 mysql_root_username: "{{ mysql_root_username_vault }}"

group_vars/stage_qa/plain.yml

@@ -341,6 +341,9 @@ harbor_token: "{{ docker_registry_token_vault }}"
 elastic_admin_username: "{{ elastic_admin_username_vault }}"
 elastic_admin_password: "{{ elastic_admin_password_vault }}"
 
+elastic_ilm_cold_min_age_threshold: 3d
+elastic_ilm_delete_min_age_threshold: 7d
+
 postgres_replicator_user_password: "{{ postgres_replicator_user_password_vault }}"
 
 mysql_root_username: "{{ mysql_root_username_vault }}"

roles/elastic/defaults/main.yaml

@@ -5,3 +5,53 @@ elastic_image_version: "7.16.1"
 
 elasticsearch_exporter_image_name: "quay.io/prometheuscommunity/elasticsearch-exporter"
 elasticsearch_exporter_image_version: "latest"
+
+# needs to be localhost due to "tolina" proxy aka 'banning all outgoing connection than 80/443'
+elastic_api_endpoint: 'localhost:{{ service_port_elasticsearch }}'
+
+elastic_default_ilm_templates:
+  - name: "{{ stage }}-ilm-default"
+    template:
+      policy:
+        phases:
+          hot:
+            min_age: 0ms
+            actions:
+              rollover:
+                max_age: '{{ elastic_ilm_hot_max_age_threshold | default("1d") }}'
+              set_priority:
+                priority: 100
+          warm:
+            min_age: '{{ elastic_ilm_warm_min_age_threshold | default("1d") }}'
+            actions:
+              readonly: {}
+              set_priority:
+                priority: 50
+              allocate:
+                number_of_replicas: 1
+          cold:
+            min_age: '{{ elastic_ilm_cold_min_age_threshold | default("7d") }}'
+            actions:
+              freeze: {}
+              readonly: {}
+              set_priority:
+                priority: 0
+              allocate:
+                number_of_replicas: 1
+          delete:
+            min_age: '{{ elastic_ilm_delete_min_age_threshold | default("30d") }}'
+            actions:
+              delete:
+                delete_searchable_snapshot: true
+
+elastic_default_index_templates:
+  - name: "{{ stage }}-idxtmpl-default"
+    template:
+      index_patterns: ["{{ stage }}-*"]
+      priority: 100
+      template:
+        settings:
+          number_of_shards: 2
+          index:
+            lifecycle:
+              name: "{{ stage }}-ilm-default"

roles/elastic/tasks/create_ilm_tmpl.yml

@@ -0,0 +1,59 @@
+---
+- name: "Set some VARs"
+  set_fact:
+    ilm_tmpl_state: "{{ ilm_tmpl.ilm_tmpl_state | default('present') }}"
+    api_object: '_ilm/policy'
+  tags:
+    - always
+
+- name: "Check if index template already exists"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ ilm_tmpl.name }}"
+    method: GET
+    status_code: [200,404]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  register: check_index_exists
+  tags:
+    - always
+
+- name: "Create Index template"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ ilm_tmpl.name }}"
+    method: PUT
+    status_code: [200]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    body: "{{ ilm_tmpl.template | to_json }} "
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  when:
+    - ilm_tmpl_state == 'present'
+    - check_index_exists.status in [404]
+  tags:
+    - always
+
+- name: "Delete Index template"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{ ilm_tmpl.name }}"
+    method: DELETE
+    status_code: [200]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    body: "{{ ilm_tmpl.template }} "
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  when:
+    - ilm_tmpl_state == 'absent'
+    - check_index_exists.status in [200]
+
+  tags:
+    - always

roles/elastic/tasks/create_index_tmpl.yml

@@ -0,0 +1,59 @@
+---
+- name: "Set some VARs"
+  set_fact:
+    idx_tmpl_state: "{{ idx_tmpl.idx_tmpl_state | default('present') }}"
+    api_object: '_index_template'
+  tags:
+    - always
+
+- name: "Check if index template already exists"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ idx_tmpl.name }}"
+    method: GET
+    status_code: [200,404]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  register: check_index_exists
+  tags:
+    - always
+
+- name: "Create Index template"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{{ idx_tmpl.name }}"
+    method: PUT
+    status_code: [200]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    body: "{{ idx_tmpl.template | to_json }} "
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  when:
+    - idx_tmpl_state == 'present'
+    - check_index_exists.status in [404]
+  tags:
+    - always
+
+- name: "Delete Index template"
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/{{ api_object }}/{ idx_tmpl.name }}"
+    method: DELETE
+    status_code: [200]
+    user: "{{ elastic_admin_username }}" 
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes 
+    body: "{{ idx_tmpl.template }} "
+    headers:
+      Content-Type: application/json
+    validate_certs: no
+  when:
+    - idx_tmpl_state == 'absent'
+    - check_index_exists.status in [200]
+
+  tags:
+    - always

roles/elastic/tasks/main.yaml

@@ -70,6 +70,7 @@
     restarted: yes
     build: no
   tags:
+    - never
     - update_certs
 
 - name: "Update {{ elastic_id }}"
@@ -80,3 +81,45 @@
   tags:
     - update_config
     - update_deployment
+
+- name: "Waiting for running elastic instance"
+  become: false
+  ansible.builtin.wait_for:
+    timeout: 180
+    port: "{{ service_port_elasticsearch }}"
+    host: localhost
+
+- name: "Check if elastic cluster API reachable"
+  become: false
+  uri:
+    url: "https://{{ elastic_api_endpoint }}/_cluster/health?pretty"
+    method: GET
+    status_code: [200]
+    user: "{{ elastic_admin_username }}"
+    password: "{{ elastic_admin_password }}"
+    force_basic_auth: yes
+    ca_path: '{{ service_base_path }}/{{ elastic_id }}/certs/ca/ca.crt'
+  register: check_elastic_api_reachable
+  delay: 5
+  retries: 15
+  until: check_elastic_api_reachable.status in [200]
+
+- name: "Configure | create default ilm templates"
+  include_tasks: create_ilm_tmpl.yml
+  loop: '{{ elastic_default_ilm_templates }}'
+  loop_control:
+    loop_var: ilm_tmpl
+  tags:
+    - elastic-ilmtmpls
+  when:
+    - inventory_hostname == groups['elastic'][0]
+
+- name: "Configure | create default index templates"
+  include_tasks: create_index_tmpl.yml
+  loop: '{{ elastic_default_index_templates }}'
+  loop_control:
+    loop_var: idx_tmpl
+  tags:
+    - elastic-idxtmpls
+  when:
+    - inventory_hostname == groups['elastic'][0]

templates/logstash/config/logstash/pipeline/filebeat.conf.j2

@@ -60,7 +60,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[fields][hostname]}-authlog-%{+YYYY.MM}"
+            index => "%{[fields][hostname]}-authlog"
+            ilm_rollover_alias => "%{[fields][hostname]}-authlog"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -72,7 +74,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "uncategorized-authlog-%{+YYYY.MM}"
+            index => "uncategorized-authlog"
+            ilm_rollover_alias => "uncategorized-authlog"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -84,7 +88,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[fields][hostname]}-syslog-%{+YYYY.MM}"
+            index => "%{[fields][hostname]}-syslog"
+            ilm_rollover_alias => "%{[fields][hostname]}-syslog"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -96,7 +102,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "uncategorized-syslog-%{+YYYY.MM}"
+            index => "uncategorized-syslog"
+            ilm_rollover_alias => "uncategorized-syslog"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -108,7 +116,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[fields][hostname]}-postgresql-%{+YYYY.MM}"
+            index => "%{[fields][hostname]}-postgresql"
+            ilm_rollover_alias => "%{[fields][hostname]}-postgresql"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -120,7 +130,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[fields][hostname]}-mysql-%{+YYYY.MM}"
+            index => "%{[fields][hostname]}-mysql"
+            ilm_rollover_alias => "%{[fields][hostname]}-mysql"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -133,7 +145,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}-%{+YYYY.MM}"
+            index => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}"
+            ilm_rollover_alias => "%{[fields][hostname]}-harbor-%{[fields][harbor-component]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -145,7 +159,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}-%{+YYYY.MM}"
+            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}"
+            ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][deployment][name]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -157,7 +173,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}-%{+YYYY.MM}"
+            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}"
+            ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][daemonset][name]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -169,7 +187,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}-%{+YYYY.MM}"
+            index => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}"
+            ilm_rollover_alias => "%{[stage]}-%{[kubernetes][namespace]}-%{[kubernetes][statefulset][name]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -181,7 +201,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "uncategorized-kubernetes-%{[kubernetes][namespace]}-%{+YYYY.MM}"
+            index => "uncategorized-kubernetes-%{[kubernetes][namespace]}"
+            ilm_rollover_alias => "uncategorized-kubernetes-%{[kubernetes][namespace]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -193,7 +215,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "%{[container][name]}-%{+YYYY.MM}"
+            index => "%{[container][name]}"
+            ilm_rollover_alias => "%{[container][name]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -205,7 +229,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM}"
+            index => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}"
+            ilm_rollover_alias => "uncategorized-%{[@metadata][beat]}-%{[@metadata][version]}"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
        }
@@ -217,7 +243,9 @@ output {
             user => "{{ elastic_admin_username }}"
             password => "{{ elastic_admin_password }}"
 
-            index => "uncategorized-%{+YYYY.MM}"
+            index => "uncategorized"
+            ilm_rollover_alias => "uncategorized"
+            ilm_policy => "{{ stage }}-ilm-default"
 
             manage_template => false
         }