From 05a6f8a67f660809e9ad9fa50234424739e3f18a Mon Sep 17 00:00:00 2001 From: Sven Ketelsen Date: Thu, 13 Jan 2022 21:55:43 +0100 Subject: [PATCH] feat: added service filtering for kibana dashboard --- create-kibana-objects.yml | 23 +++++++------ roles/elastic/handlers/main.yml | 1 - roles/elastic/meta/main.yml | 1 - .../smardigo_default_objects.json.j2 | 3 -- roles/kibana/defaults/main.yaml | 4 +++ .../tasks/_configure_dashboards.yml | 13 +++----- .../tasks/_configure_indexpattern.yml | 0 .../tasks/_configure_roles.yml | 0 .../tasks/_configure_searches.yml | 0 .../tasks/_configure_spaces.yml | 33 +++++++++++++++---- .../tasks/_configure_users.yml | 0 .../tasks/_import_savedobjects.yml | 16 ++++++--- .../templates/smardigo_dashboard.json.j2 | 25 ++++++++++++++ .../smardigo_index_pattern_service.json.j2 | 17 ++++++++++ .../smardigo_index_pattern_tenant.json.j2 | 17 ++++++++++ .../kibana/templates/smardigo_search.json.j2 | 26 +++++++++++++++ 16 files changed, 144 insertions(+), 35 deletions(-) delete mode 100644 roles/elastic/handlers/main.yml delete mode 100644 roles/elastic/meta/main.yml delete mode 100644 roles/elastic/templates/smardigo_default_objects.json.j2 rename roles/{elastic => kibana}/tasks/_configure_dashboards.yml (95%) rename roles/{elastic => kibana}/tasks/_configure_indexpattern.yml (100%) rename roles/{elastic => kibana}/tasks/_configure_roles.yml (100%) rename roles/{elastic => kibana}/tasks/_configure_searches.yml (100%) rename roles/{elastic => kibana}/tasks/_configure_spaces.yml (69%) rename roles/{elastic => kibana}/tasks/_configure_users.yml (100%) rename roles/{elastic => kibana}/tasks/_import_savedobjects.yml (66%) create mode 100644 roles/kibana/templates/smardigo_dashboard.json.j2 create mode 100644 roles/kibana/templates/smardigo_index_pattern_service.json.j2 create mode 100644 roles/kibana/templates/smardigo_index_pattern_tenant.json.j2 create mode 100644 roles/kibana/templates/smardigo_search.json.j2 diff --git a/create-kibana-objects.yml b/create-kibana-objects.yml index bc79d68..0270817 100644 --- a/create-kibana-objects.yml +++ b/create-kibana-objects.yml @@ -126,14 +126,16 @@ - actions - osquery - savedObjectsTagging - es_indexpattern_name: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-*' + es_index_pattern_tenant: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-*' + es_index_pattern_service: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-*-{{ cluster_service }}-*' es_search_name: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-{{ cluster_service }}' es_dashboard_name: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-{{ cluster_service }}' + es_container_name: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-01-{{ cluster_service }}' tasks: - name: "Do some stuff in elastic with spaces ... " include_role: - name: elastic + name: kibana tasks_from: _configure_spaces.yml apply: tags: @@ -144,9 +146,9 @@ tags: - es-spaces - - name: "Do some stuff in elastic with roles ... " + - name: "Do some stuff in elastic with roles ..." include_role: - name: elastic + name: kibana tasks_from: _configure_roles.yml apply: tags: @@ -157,9 +159,9 @@ tags: - es-roles - - name: "Do some stuff in elastic with users ... " + - name: "Do some stuff in elastic with users ..." include_role: - name: elastic + name: kibana tasks_from: _configure_users.yml apply: tags: @@ -170,9 +172,9 @@ tags: - es-users - - name: "Do some stuff in elastic with spaces ... " + - name: "Do some stuff in elastic with spaces ..." include_role: - name: elastic + name: kibana tasks_from: _import_savedobjects.yml apply: tags: @@ -180,12 +182,13 @@ vars: es_space: *es_space_name es_indexpattern_title: '{{ es_indexpattern_name }}' - es_indexpattern_uuid: '{{ es_indexpattern_name | to_uuid }}' + es_index_pattern_tenant_uuid: '{{ es_index_pattern_tenant | to_uuid }}' + es_index_pattern_service_uuid: '{{ es_index_pattern_service | to_uuid }}' es_search_title: '{{ es_search_name }}' es_search_uuid: '{{ es_search_name | to_uuid }}' es_panel_uuid: "{{ 'panel_' + es_dashboard_name | to_uuid }}" es_dashboard_title: '{{ es_dashboard_name }}' - es_dashboard_uuid: '{{ es_dashboard_name | to_uuid }}' + es_dashboard_uuid: '{{ es_dashboard_name | to_uuid }}' tags: - es-importobjects diff --git a/roles/elastic/handlers/main.yml b/roles/elastic/handlers/main.yml deleted file mode 100644 index ed97d53..0000000 --- a/roles/elastic/handlers/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/roles/elastic/meta/main.yml b/roles/elastic/meta/main.yml deleted file mode 100644 index ed97d53..0000000 --- a/roles/elastic/meta/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/roles/elastic/templates/smardigo_default_objects.json.j2 b/roles/elastic/templates/smardigo_default_objects.json.j2 deleted file mode 100644 index 168077b..0000000 --- a/roles/elastic/templates/smardigo_default_objects.json.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{"attributes":{"fieldAttrs":"{\"message_full\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"{{ es_indexpattern_title }}","typeMeta":"{}"},"coreMigrationVersion":"7.16.1","id":"{{ es_indexpattern_uuid }}","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern"} -{"attributes":{"columns":["message_full"],"description":"","grid":{},"hideChart":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"{{ es_search_title }}"},"coreMigrationVersion":"7.16.1","id":"{{ es_search_uuid }}","migrationVersion":{"search":"7.9.3"},"references":[{"id":"{{ es_indexpattern_uuid }}","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.16.1\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":0,\"w\":50,\"h\":50,\"i\":\"{{ es_panel_uuid }}\"},\"panelIndex\":\"{{ es_panel_uuid }}\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_{{ es_panel_uuid }}\"}]","timeRestore":false,"title":"{{ es_dashboard_title }}","version":1},"coreMigrationVersion":"7.16.1","id":"{{ es_dashboard_uuid }}","migrationVersion":{"dashboard":"7.16.0"},"references":[{"id":"{{ es_search_uuid }}","name":"{{ es_panel_uuid }}:panel_{{ es_panel_uuid }}","type":"search"}],"type":"dashboard"} diff --git a/roles/kibana/defaults/main.yaml b/roles/kibana/defaults/main.yaml index 1284057..de91c04 100644 --- a/roles/kibana/defaults/main.yaml +++ b/roles/kibana/defaults/main.yaml @@ -2,3 +2,7 @@ kibana_image_name: "docker.elastic.co/kibana/kibana" kibana_image_version: "7.16.1" + +kibana_advanced_settings: + changes: + truncate:maxHeight: 0 \ No newline at end of file diff --git a/roles/elastic/tasks/_configure_dashboards.yml b/roles/kibana/tasks/_configure_dashboards.yml similarity index 95% rename from roles/elastic/tasks/_configure_dashboards.yml rename to roles/kibana/tasks/_configure_dashboards.yml index db3543f..ff7b8bc 100644 --- a/roles/elastic/tasks/_configure_dashboards.yml +++ b/roles/kibana/tasks/_configure_dashboards.yml @@ -46,7 +46,7 @@ url: "https://{{ api_endpoint }}/s/{{ es_space }}/api/saved_objects/_find?per_page=10000&type=search" method: GET status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes register: all_searches @@ -84,7 +84,7 @@ name: '{{ panelindex_uuid }}' delegate_to: localhost type: 'search' - id: '{{ lookup_search_object[0].id }}' + id: '{{ lookup_search_object[0].id }}' when: - lookup_search_object | length > 0 when: @@ -107,16 +107,13 @@ - debug: msg: 'DEBUG elastic_dashboard_cleaned: {{ elastic_dashboard_cleaned }}' - - - - name: "Create {{ es_object_type }} <<{{ elastic_dashboard.attributes.title }}>>" delegate_to: localhost uri: url: "https://{{ api_endpoint }}{{ api_path }}/{{ es_object_type }}" method: POST status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: @@ -135,7 +132,7 @@ url: 'https://{{ api_endpoint }}{{ api_path }}/{{ es_object_type }}/{{ lookup_dashboard_object[0]["id"] }}' method: PUT status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: @@ -154,7 +151,7 @@ url: 'https://{{ api_endpoint }}{{ api_path }}/{{ es_object_type }}/{{ lookup_dashboard_object[0]["id"] }}' method: DELETE status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: diff --git a/roles/elastic/tasks/_configure_indexpattern.yml b/roles/kibana/tasks/_configure_indexpattern.yml similarity index 100% rename from roles/elastic/tasks/_configure_indexpattern.yml rename to roles/kibana/tasks/_configure_indexpattern.yml diff --git a/roles/elastic/tasks/_configure_roles.yml b/roles/kibana/tasks/_configure_roles.yml similarity index 100% rename from roles/elastic/tasks/_configure_roles.yml rename to roles/kibana/tasks/_configure_roles.yml diff --git a/roles/elastic/tasks/_configure_searches.yml b/roles/kibana/tasks/_configure_searches.yml similarity index 100% rename from roles/elastic/tasks/_configure_searches.yml rename to roles/kibana/tasks/_configure_searches.yml diff --git a/roles/elastic/tasks/_configure_spaces.yml b/roles/kibana/tasks/_configure_spaces.yml similarity index 69% rename from roles/elastic/tasks/_configure_spaces.yml rename to roles/kibana/tasks/_configure_spaces.yml index 0458d2d..91f58e3 100644 --- a/roles/elastic/tasks/_configure_spaces.yml +++ b/roles/kibana/tasks/_configure_spaces.yml @@ -10,14 +10,14 @@ url: "https://{{ api_endpoint }}{{ api_path }}" method: GET status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes register: all_spaces become: false - set_fact: - lookup_space_object: '{{ all_spaces.json | community.general.json_query(spaces_query) }}' + lookup_space_object: "{{ all_spaces.json | community.general.json_query(spaces_query) }}" vars: spaces_query: "[?name=='{{ elastic_space.name }}']" @@ -28,12 +28,12 @@ - set_fact: elastic_space_cleaned: "{{ elastic_space_cleaned | combine({item.key: item.value}) }}" - with_dict: '{{ elastic_space }}' + with_dict: "{{ elastic_space }}" when: - item.key not in ['elastic_state'] - debug: - msg: '{{ lookup_space_object | to_json }}' + msg: "{{ lookup_space_object | to_json }}" - name: "Create space <<{{ elastic_space.name }}>>" delegate_to: localhost @@ -41,7 +41,7 @@ url: "https://{{ api_endpoint }}{{ api_path }}" method: POST status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: @@ -60,7 +60,7 @@ url: "https://{{ api_endpoint }}{{ api_path }}/{{ elastic_space.name }}" method: PUT status_code: [200] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: @@ -73,13 +73,32 @@ - space_exists - elastic_space.elastic_state == 'present' +- name: "Update space advanced settings <<{{ elastic_space.name }}>>" + delegate_to: localhost + uri: + url: "https://{{ api_endpoint }}/s/{{ elastic_space.name }}/api/kibana/settings" + method: POST + status_code: [200] + user: "{{ elastic_admin_username_vault }}" + password: "{{ elastic_admin_password_vault }}" + force_basic_auth: yes + headers: + Content-Type: application/json + kbn-xsrf: true + body_format: json + body: '{{ kibana_advanced_settings | to_json }}' + become: false + when: + - space_exists + - elastic_space.elastic_state == 'present' + - name: "DELETE space <<{{ elastic_space.name }}>>" delegate_to: localhost uri: url: "https://{{ api_endpoint }}{{ api_path }}/{{ elastic_space.name }}" method: DELETE status_code: [204] - user: "{{ elastic_admin_username_vault }}" + user: "{{ elastic_admin_username_vault }}" password: "{{ elastic_admin_password_vault }}" force_basic_auth: yes headers: diff --git a/roles/elastic/tasks/_configure_users.yml b/roles/kibana/tasks/_configure_users.yml similarity index 100% rename from roles/elastic/tasks/_configure_users.yml rename to roles/kibana/tasks/_configure_users.yml diff --git a/roles/elastic/tasks/_import_savedobjects.yml b/roles/kibana/tasks/_import_savedobjects.yml similarity index 66% rename from roles/elastic/tasks/_import_savedobjects.yml rename to roles/kibana/tasks/_import_savedobjects.yml index e33e79c..2bd74a5 100644 --- a/roles/elastic/tasks/_import_savedobjects.yml +++ b/roles/kibana/tasks/_import_savedobjects.yml @@ -2,10 +2,13 @@ - set_fact: api_path: '/s/{{ es_space }}/api/saved_objects' -- name: "Import smardigo default dashboard and its related objects (index-pattern,search)" +- name: "Import smardigo default dashboard and its related objects (index-pattern,search,...)" delegate_to: localhost set_fact: - es_import_objects: "{{ lookup('template','smardigo_default_objects.json.j2') }}" + es_object_smardigo_index_pattern_tenant: "{{ lookup('template','smardigo_index_pattern_tenant.json.j2') }}" + es_object_smardigo_index_pattern_service: "{{ lookup('template','smardigo_index_pattern_service.json.j2') }}" + es_object_smardigo_search: "{{ lookup('template','smardigo_search.json.j2') }}" + es_object_smardigo_dashboard: "{{ lookup('template','smardigo_dashboard.json.j2') }}" when: - elastic_state == 'present' @@ -13,7 +16,11 @@ delegate_to: localhost copy: dest: '/tmp/es_objects_ready_to_import__objects.ndjson' - content: '{{ es_import_objects }}' + content: | + {{ es_object_smardigo_index_pattern_tenant | to_json(separators=(',',':')) }} + {{ es_object_smardigo_index_pattern_service | to_json(separators=(',',':')) }} + {{ es_object_smardigo_search | to_json(separators=(',',':')) }} + {{ es_object_smardigo_dashboard | to_json(separators=(',',':')) }} when: - elastic_state == 'present' @@ -45,7 +52,6 @@ when: - elastic_state == 'present' - - name: "Set default indexpattern ..." delegate_to: localhost uri: @@ -60,7 +66,7 @@ body_format: json body: changes: - defaultIndex: '{{ es_indexpattern_uuid }}' + defaultIndex: '{{ es_index_pattern_tenant_uuid }}' become: false when: - elastic_state == 'present' diff --git a/roles/kibana/templates/smardigo_dashboard.json.j2 b/roles/kibana/templates/smardigo_dashboard.json.j2 new file mode 100644 index 0000000..c40849e --- /dev/null +++ b/roles/kibana/templates/smardigo_dashboard.json.j2 @@ -0,0 +1,25 @@ +{ + "attributes" : { + "description" : "", + "hits" : 0, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "optionsJSON" : "{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}", + "panelsJSON" : "[{\"version\":\"7.16.1\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":0,\"w\":50,\"h\":50,\"i\":\"{{ es_panel_uuid }}\"},\"panelIndex\":\"{{ es_panel_uuid }}\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_{{ es_panel_uuid }}\"}]", + "timeRestore" : false, + "title" : "{{ es_dashboard_title }}", + "version" : 1 + }, + "coreMigrationVersion" : "7.16.1", + "id" : "{{ es_dashboard_uuid }}", + "migrationVersion" : { + "dashboard" : "7.16.0" + }, + "references" : [ { + "id" : "{{ es_search_uuid }}", + "name" : "{{ es_panel_uuid }}:panel_{{ es_panel_uuid }}", + "type" : "search" + } ], + "type" : "dashboard" +} \ No newline at end of file diff --git a/roles/kibana/templates/smardigo_index_pattern_service.json.j2 b/roles/kibana/templates/smardigo_index_pattern_service.json.j2 new file mode 100644 index 0000000..4c1c464 --- /dev/null +++ b/roles/kibana/templates/smardigo_index_pattern_service.json.j2 @@ -0,0 +1,17 @@ +{ + "attributes" : { + "fieldAttrs" : "{\"message_full\":{\"count\":1}}", + "fields" : "[]", + "runtimeFieldMap" : "{}", + "timeFieldName" : "@timestamp", + "title" : "{{ es_index_pattern_service }}", + "typeMeta" : "{}" + }, + "coreMigrationVersion" : "7.16.1", + "id" : "{{ es_index_pattern_service_uuid }}", + "migrationVersion" : { + "index-pattern" : "7.11.0" + }, + "references" : [ ], + "type" : "index-pattern" +} \ No newline at end of file diff --git a/roles/kibana/templates/smardigo_index_pattern_tenant.json.j2 b/roles/kibana/templates/smardigo_index_pattern_tenant.json.j2 new file mode 100644 index 0000000..8736ebe --- /dev/null +++ b/roles/kibana/templates/smardigo_index_pattern_tenant.json.j2 @@ -0,0 +1,17 @@ +{ + "attributes" : { + "fieldAttrs" : "{\"message_full\":{\"count\":1}}", + "fields" : "[]", + "runtimeFieldMap" : "{}", + "timeFieldName" : "@timestamp", + "title" : "{{ es_index_pattern_tenant }}", + "typeMeta" : "{}" + }, + "coreMigrationVersion" : "7.16.1", + "id" : "{{ es_index_pattern_tenant_uuid }}", + "migrationVersion" : { + "index-pattern" : "7.11.0" + }, + "references" : [ ], + "type" : "index-pattern" +} \ No newline at end of file diff --git a/roles/kibana/templates/smardigo_search.json.j2 b/roles/kibana/templates/smardigo_search.json.j2 new file mode 100644 index 0000000..73b248f --- /dev/null +++ b/roles/kibana/templates/smardigo_search.json.j2 @@ -0,0 +1,26 @@ +{ + "attributes" : { + "columns" : [ "message_full" ], + "description" : "", + "grid" : { + }, + "hideChart" : false, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"{{ es_container_name }}\"}},\"query\":{\"match_phrase\":{\"container.name\":\"{{ es_container_name }}\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort" : [ [ "@timestamp", + "desc" ] ], + "title" : "{{ es_search_title }}" + }, + "coreMigrationVersion" : "7.16.1", + "id" : "{{ es_search_uuid }}", + "migrationVersion" : { + "search" : "7.9.3" + }, + "references" : [ { + "id" : "{{ es_index_pattern_service_uuid }}", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", + "type" : "index-pattern" + } ], + "type" : "search" +} \ No newline at end of file