hetzner-ansible/patchday.yml

---

### tags:
###   check_elastic_cluster

- hosts: elastic
  serial: 1
  become: yes
  tasks:
    - name: "Smardigo Patchday: update pkgs"
      ansible.builtin.apt:
        upgrade: yes
        update_cache: yes
        autoremove: yes
        autoclean: yes

    - name: "Smardigo Patchday: find docker_compose.yml files"
      ansible.builtin.find:
        paths: '{{ service_base_path }}'
        pattern: 'docker*.yml'
        recurse: yes
      register: docker_compose_services

    - name: "Smardigo Patchday: shutdown services"
      community.docker.docker_compose:
        project_src: '{{ item.path | dirname }}'
        state: absent
      loop: '{{ docker_compose_services.files }}'

    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
      ansible.builtin.reboot:
        post_reboot_delay: 30
        reboot_timeout: 300

    - name: "Smardigo Patchday: wait_for host after reboot"
      become: no
      delegate_to: localhost
      ansible.builtin.wait_for:
        delay: 15
        timeout: 180
        port: 22
        host: '{{ stage_server_ip }}'
        search_regex: OpenSSH

    - name: "Smardigo Patchday: start services"
      community.docker.docker_compose:
        project_src: '{{ item.path | dirname }}'
        state: present
      loop: '{{ docker_compose_services.files }}'

    - name: "Smardigo Patchday: wait until cluster is green"
      ansible.builtin.uri:
        url: "https://localhost:9200/_cluster/health"
        user: "{{ elastic_admin_username }}"
        password: "{{ elastic_admin_password }}"
        force_basic_auth: true
        status_code: 200
        ca_path: "{{ service_base_path }}/{{ elastic_id }}/certs/ca/ca.crt"
      register: check_elastic_cluster
      delay: 30
      retries: 30
      until:
        - check_elastic_cluster.json is defined
        - check_elastic_cluster.json.status == 'green'
      no_log: true
      tags:
        - check_elastic_cluster

- hosts: postgres
  serial: 1
  become: yes
  tasks:
    - name: "Smardigo Patchday: stop service(s)"
      ansible.builtin.systemd:
        name: postgresql
        state: stopped

    - name: "Smardigo Patchday: update pkgs"
      ansible.builtin.apt:
        upgrade: yes
        update_cache: yes
        autoremove: yes
        autoclean: yes

    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
      ansible.builtin.reboot:
        post_reboot_delay: 30
        reboot_timeout: 300

    - name: "Smardigo Patchday: wait_for host after reboot"
      become: no
      delegate_to: localhost
      ansible.builtin.wait_for:
        delay: 15
        timeout: 180
        port: 22
        host: '{{ stage_server_ip }}'
        search_regex: OpenSSH

    - name: "Smardigo Patchday: start services"
      ansible.builtin.systemd:
        name: postgresql
        state: started

- hosts: all,!elastic,!postgres,!k8s_cluster,!gw
  serial: 10
  become: yes
  tasks:
    - name: "Smardigo Patchday: update pkgs"
      ansible.builtin.apt:
        upgrade: yes
        update_cache: yes
        autoremove: yes
        autoclean: yes

    - name: "Smardigo Patchday: find docker_compose.yml files"
      ansible.builtin.find:
        paths: '{{ service_base_path }}'
        pattern: 'docker*.yml'
        recurse: yes
      register: docker_compose_services

    - name: "Smardigo Patchday: shutdown services"
      community.docker.docker_compose:
        project_src: '{{ item.path | dirname }}'
        state: absent
      loop: '{{ docker_compose_services.files }}'

    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
      ansible.builtin.reboot:
        post_reboot_delay: 30
        reboot_timeout: 300

    - name: "Smardigo Patchday: wait_for host after reboot"
      become: no
      delegate_to: localhost
      ansible.builtin.wait_for:
        delay: 15
        timeout: 180
        port: 22
        host: '{{ stage_server_ip }}'
        search_regex: OpenSSH

    - name: "Smardigo Patchday: start services"
      community.docker.docker_compose:
        project_src: '{{ item.path | dirname }}'
        state: present
      loop: '{{ docker_compose_services.files }}'

- hosts: k8s_cluster
  serial: 1
  become: yes

  tasks:
      # draining the hard way
      # due to force( delete static pods) + relative short terminate_grace_period +
      # --delete-local-data to kick pods with emptyDir
      #
      # ATTENTION: needs to be done via command instead of kubernetes module
      #            due to missing flag for --delete-emptydir-data
      #            ¯\_(ツ)_/¯
    - name: "Smardigo Patchday: drain node"
      delegate_to: "{{ groups['kube_control_plane'][0] }}"
      ansible.builtin.command: "/usr/local/bin/kubectl drain --timeout 2m --ignore-daemonsets --force --delete-emptydir-data {{ inventory_hostname | lower }}"
      register: node_drained
      until: node_drained
      retries: 3
      delay: 30
      failed_when: false

    - name: "Smardigo Patchday: stop k8s basic services"
      ansible.builtin.systemd:
        name: '{{ item }}'
        state: stopped
      loop: '{{ k8s_basic_services }}'

    - name: "Smardigo Patchday: update pkgs"
      ansible.builtin.apt:
        autoclean: yes
        autoremove: yes
        update_cache: yes
        upgrade: yes

    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
      ansible.builtin.reboot:
        post_reboot_delay: 30
        reboot_timeout: 300

    - name: "Smardigo Patchday: wait_for host after reboot"
      become: no
      delegate_to: localhost
      ansible.builtin.wait_for:
        delay: 15
        timeout: 180
        port: 22
        host: '{{ stage_server_ip }}'
        search_regex: OpenSSH

    - name: "Smardigo Patchday: start k8s basic services"
      ansible.builtin.systemd:
        name: '{{ item }}'
        state: started
      loop: '{{ k8s_basic_services }}'

    - name: "Smardigo Patchday: wait for node readiness"
      delegate_to: "{{ groups['kube_control_plane'][0] }}"
      kubernetes.core.k8s:
        kind: Node
        state: present
        name: '{{ inventory_hostname | lower }}'
        wait_condition:
          reason: KubeletReady
          type: Ready
          status: True
        wait_timeout: 120
      retries: 5
      delay: 10

    - name: "Smardigo Patchday: uncordon node"
      delegate_to: "{{ groups['kube_control_plane'][0] }}"
      kubernetes.core.k8s_drain:
        state: uncordon
        name: '{{ inventory_hostname }}'