From 685eef3a5cf76a42c35fb373b6a268244c03db59 Mon Sep 17 00:00:00 2001 From: Sven Ketelsen Date: Wed, 28 Jun 2023 10:37:52 +0200 Subject: [PATCH] DEV-1114 patchday: extracted harbor/maria in own stages --- patchday.yml | 127 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/patchday.yml b/patchday.yml index 52b6245..370fe0c 100644 --- a/patchday.yml +++ b/patchday.yml @@ -5,7 +5,7 @@ - hosts: prometheus vars: - start: '{{ ansible_date_time.epoch }}' + start: "{{ ansible_date_time.epoch }}" tasks: - name: "DO some stuff for silencing" @@ -22,8 +22,8 @@ - name: job value: .+ isRegex: true - startsAt: '{{ silence_starts_at }}' - endsAt: '{{ silence_ends_at }}' + startsAt: "{{ silence_starts_at }}" + endsAt: "{{ silence_ends_at }}" createdBy: patchday-automatism comment: patchday id: @@ -38,15 +38,15 @@ headers: Content-Type: application/json body_format: json - body: '{{ silence | to_json }}' + body: "{{ silence | to_json }}" rescue: - name: "Rescue silencing - sending mail to DEVOPS-DL" - delegate_to: '{{ stage }}-mail-01' + delegate_to: "{{ stage }}-mail-01" community.general.mail: host: localhost port: 25 - to: '{{ devops_email_address }}' + to: "{{ devops_email_address }}" subject: "patchday( {{ lookup('pipe','date +%Y-%m-%d_%H:%M') }} ) problem report for failed silencing" body: | Dear Sir or Madam, @@ -59,6 +59,50 @@ your automation-bofh +- hosts: harbor + serial: 1 + become: yes + tasks: + - name: "Smardigo Patchday: update pkgs" + ansible.builtin.apt: + upgrade: yes + update_cache: yes + autoremove: yes + autoclean: yes + + - name: "Smardigo Patchday: find docker_compose.yml files" + ansible.builtin.find: + paths: "{{ service_base_path }}" + pattern: 'docker*.yml' + recurse: yes + register: docker_compose_services + + - name: "Smardigo Patchday: shutdown services" + community.docker.docker_compose: + project_src: "{{ item | dirname }}" + state: absent + loop: "{{ docker_compose_services.files | map(attribute='path') | select('match', '.*/'+stage+'-.*') }}" + + - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" + ansible.builtin.reboot: + post_reboot_delay: 30 + reboot_timeout: 300 + + - name: "Smardigo Patchday: wait_for host after reboot" + become: no + delegate_to: localhost + ansible.builtin.wait_for: + delay: 15 + timeout: 180 + port: 22 + host: "{{ stage_server_ip }}" + search_regex: OpenSSH + + - name: "Smardigo Patchday: start services" + community.docker.docker_compose: + project_src: "{{ item | dirname }}" + state: present + loop: "{{ docker_compose_services.files | map(attribute='path') }}" - hosts: elastic serial: 1 @@ -73,16 +117,16 @@ - name: "Smardigo Patchday: find docker_compose.yml files" ansible.builtin.find: - paths: '{{ service_base_path }}' + paths: "{{ service_base_path }}" pattern: 'docker*.yml' recurse: yes register: docker_compose_services - name: "Smardigo Patchday: shutdown services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item | dirname }}" state: absent - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files | map(attribute='path') | select('match', '.*/'+stage+'-.*') }}" - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" ansible.builtin.reboot: @@ -96,14 +140,14 @@ delay: 15 timeout: 180 port: 22 - host: '{{ stage_server_ip }}' + host: "{{ stage_server_ip }}" search_regex: OpenSSH - name: "Smardigo Patchday: start services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item.path | dirname }}" state: present - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files }}" - name: "Smardigo Patchday: wait until cluster is green" ansible.builtin.uri: @@ -151,7 +195,7 @@ delay: 15 timeout: 180 port: 22 - host: '{{ stage_server_ip }}' + host: "{{ stage_server_ip }}" search_regex: OpenSSH - name: "Open and mount LUKS encrypted LVM for datadir" @@ -187,7 +231,7 @@ community.postgresql.postgresql_ping: port: 5432 ssl_mode: require - login_host: '{{ stage_private_server_ip }}' + login_host: "{{ stage_private_server_ip }}" register: check_postgres ignore_errors: true @@ -206,18 +250,18 @@ community.postgresql.postgresql_ping: port: 5432 ssl_mode: require - login_host: '{{ stage_private_server_ip }}' + login_host: "{{ stage_private_server_ip }}" register: check_postgres_again retries: 5 failed_when: not check_postgres_again.is_available rescue: - name: "Smardigo Patchday: error-handling - send mail to DEVOPS-DL" - delegate_to: '{{ stage }}-mail-01' + delegate_to: "{{ stage }}-mail-01" community.general.mail: host: localhost port: 25 - to: '{{ devops_email_address }}' + to: "{{ devops_email_address }}" subject: "patchday( {{ lookup('pipe', 'date +%Y-%m-%d_%H:%M') }} ) problem report for {{ inventory_hostname }}" body: | Dear Sir or Madam, @@ -245,16 +289,16 @@ - name: "Smardigo Patchday: find docker_compose.yml files" ansible.builtin.find: - paths: '{{ service_base_path }}' + paths: "{{ service_base_path }}" pattern: 'docker*.yml' recurse: true register: docker_compose_services - name: "Smardigo Patchday: shutdown services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item | dirname }}" state: absent - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files | map(attribute='path') | select('match', '.*/'+stage+'-.*') }}" - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" ansible.builtin.reboot: @@ -268,16 +312,16 @@ delay: 15 timeout: 180 port: 22 - host: '{{ stage_server_ip }}' + host: "{{ stage_server_ip }}" search_regex: OpenSSH - name: "Smardigo Patchday: start services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item | dirname }}" state: present - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files | map(attribute='path') | select('match', '.*/'+stage+'-.*') }}" -- hosts: all,!elastic,!postgres,!k8s_cluster,!iam,!restore,!keycloak +- hosts: all,!harbor,!elastic,!postgres,!iam,!keycloak,!k8s_cluster,!restore serial: 10 become: yes tasks: @@ -296,16 +340,16 @@ - name: "Smardigo Patchday: find docker_compose.yml files" ansible.builtin.find: - paths: '{{ service_base_path }}' + paths: "{{ service_base_path }}" pattern: 'docker*.yml' recurse: yes register: docker_compose_services - name: "Smardigo Patchday: shutdown services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item.path | dirname }}" state: absent - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files }}" - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" ansible.builtin.reboot: @@ -319,14 +363,14 @@ delay: 15 timeout: 180 port: 22 - host: '{{ stage_server_ip }}' + host: "{{ stage_server_ip }}" search_regex: OpenSSH - name: "Smardigo Patchday: start services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item.path | dirname }}" state: present - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files }}" - name: "Ensure SMA-portal is up and running" # there is a hard dependency within SMA-portal (VM: <>-management-01) during application start process @@ -366,20 +410,19 @@ - name: "Smardigo Patchday: SMA-portal not reachable - shutdown services" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item.path | dirname }}" state: absent - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files }}" - name: "Smardigo Patchday: SMA-portal not reachable - start services again" community.docker.docker_compose: - project_src: '{{ item.path | dirname }}' + project_src: "{{ item.path | dirname }}" state: present - loop: '{{ docker_compose_services.files }}' + loop: "{{ docker_compose_services.files }}" when: - "'management' in inventory_hostname" - - hosts: k8s_cluster serial: 1 become: yes @@ -403,9 +446,9 @@ - name: "Smardigo Patchday: stop k8s basic services" ansible.builtin.systemd: - name: '{{ item }}' + name: "{{ item }}" state: stopped - loop: '{{ k8s_basic_services }}' + loop: "{{ k8s_basic_services }}" - name: "Smardigo Patchday: update pkgs" ansible.builtin.apt: @@ -426,21 +469,21 @@ delay: 15 timeout: 180 port: 22 - host: '{{ stage_server_ip }}' + host: "{{ stage_server_ip }}" search_regex: OpenSSH - name: "Smardigo Patchday: start k8s basic services" ansible.builtin.systemd: - name: '{{ item }}' + name: "{{ item }}" state: started - loop: '{{ k8s_basic_services }}' + loop: "{{ k8s_basic_services }}" - name: "Smardigo Patchday: wait for node readiness" delegate_to: "{{ groups['kube_control_plane'][0] }}" kubernetes.core.k8s: kind: Node state: present - name: '{{ inventory_hostname | lower }}' + name: "{{ inventory_hostname | lower }}" wait_condition: reason: KubeletReady type: Ready @@ -453,4 +496,4 @@ delegate_to: "{{ groups['kube_control_plane'][0] }}" kubernetes.core.k8s_drain: state: uncordon - name: '{{ inventory_hostname }}' + name: "{{ inventory_hostname }}"