DEV-342: added patchday playbook

4 years ago · 59a409fc32
parent ab22c2359b
commit 59a409fc32
6 changed files with 254 additions and 5 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -11,6 +11,7 @@ services:
 stages:
  - ansible-lint
  - ansible-builder
+  - ansible-playbook

 ansible-lint-job:
  stage: ansible-lint
@ -37,3 +38,31 @@ ansible-builder-job:
  tags:
    - dind
    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-patchday-dev:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-playbook
+  script:
+    - export HETZNER_LABEL_SELECTOR='stage=dev'
+    - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass
+    - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster
+  after_script:
+    - rm /tmp/vault-pass
+  when: manual
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-patchday-qa:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-playbook
+  script:
+    - export HETZNER_LABEL_SELECTOR='stage=qa'
+    - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass
+    - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster
+  after_script:
+    - rm /tmp/vault-pass
+  when: manual
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
--- a/ansible.cfg
+++ b/ansible.cfg
@ -1,4 +1,6 @@
 [defaults]
+pipelining = True
+host_key_checking = False
 inventory_plugins = ./inventory_plugins
 callbacks_enabled = timer
 interpreter_python = auto_silent
--- a/group_vars/all/services.yml
+++ b/group_vars/all/services.yml
@ -0,0 +1,4 @@
+---
+
+elastic_id: "{{ inventory_hostname }}-elastic"
+elastic_exporter_id: "{{ inventory_hostname }}-elastic-exporter"
--- a/patchday.yml
+++ b/patchday.yml
@ -0,0 +1,218 @@
+---
+
+### tags:
+###   check_elastic_cluster
+
+- hosts: elastic
+  serial: 1
+  tasks:
+    - name: "Smardigo Patchday: update pkgs"
+      ansible.builtin.apt:
+        upgrade: yes
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+
+    - name: "Smardigo Patchday: find docker_compose.yml files"
+      ansible.builtin.find:
+        paths: '{{ service_base_path }}'
+        pattern: 'docker*.yml'
+        recurse: yes
+      register: docker_compose_services
+
+    - name: "Smardigo Patchday: shutdown services"
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: absent
+      loop: '{{ docker_compose_services.files }}'
+
+    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
+      ansible.builtin.reboot:
+        post_reboot_delay: 30
+        reboot_timeout: 60
+
+    - name: "Smardigo Patchday: wait_for host after reboot"
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        delay: 15
+        timeout: 180
+        port: 22
+        host: '{{ stage_server_ip }}'
+        search_regex: OpenSSH
+
+    - name: "Smardigo Patchday: start services"
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: present
+      loop: '{{ docker_compose_services.files }}'
+
+    - name: "Smardigo Patchday: wait until cluster is green"
+      ansible.builtin.uri:
+        url: "https://localhost:9200/_cluster/health"
+        user: "{{ elastic_admin_username }}"
+        password: "{{ elastic_admin_password }}"
+        force_basic_auth: true
+        status_code: 200
+        ca_path: "{{ service_base_path }}/{{ elastic_id }}/certs/ca/ca.crt"
+      register: check_elastic_cluster
+      delay: 30
+      retries: 30
+      until:
+        - check_elastic_cluster.json is defined
+        - check_elastic_cluster.json.status == 'green'
+      no_log: true
+      tags:
+        - check_elastic_cluster
+
+- hosts: postgres
+  serial: 1
+  tasks:
+    - name: "Smardigo Patchday: stop service(s)"
+      ansible.builtin.systemd:
+        name: postgresql
+        state: stopped
+
+    - name: "Smardigo Patchday: update pkgs"
+      ansible.builtin.apt:
+        upgrade: yes
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+
+    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
+      ansible.builtin.reboot:
+        post_reboot_delay: 30
+        reboot_timeout: 60
+
+    - name: "Smardigo Patchday: wait_for host after reboot"
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        delay: 15
+        timeout: 180
+        port: 22
+        host: '{{ stage_server_ip }}'
+        search_regex: OpenSSH
+
+    - name: "Smardigo Patchday: start services"
+      ansible.builtin.systemd:
+        name: postgresql
+        state: started
+
+- hosts: all,!elastic,!postgres,!k8s_cluster,!gw
+  serial: 10
+  tasks:
+    - name: "Smardigo Patchday: update pkgs"
+      ansible.builtin.apt:
+        upgrade: yes
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+
+    - name: "Smardigo Patchday: find docker_compose.yml files"
+      ansible.builtin.find:
+        paths: '{{ service_base_path }}'
+        pattern: 'docker*.yml'
+        recurse: yes
+      register: docker_compose_services
+
+    - name: "Smardigo Patchday: shutdown services"
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: absent
+      loop: '{{ docker_compose_services.files }}'
+
+    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
+      ansible.builtin.reboot:
+        post_reboot_delay: 30
+        reboot_timeout: 60
+
+    - name: "Smardigo Patchday: wait_for host after reboot"
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        delay: 15
+        timeout: 180
+        port: 22
+        host: '{{ stage_server_ip }}'
+        search_regex: OpenSSH
+
+    - name: "Smardigo Patchday: start services"
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: present
+      loop: '{{ docker_compose_services.files }}'
+
+- hosts: k8s_cluster
+  serial: 1
+  vars:
+    k8s_basic_services:
+      - kubelet
+      - docker
+  tasks:
+      # draining the hard way
+      # due to force( delete static pods) + relative short terminate_grace_period +
+      # --delete-local-data to kick pods with emptyDir
+      #
+      # ATTENTION: needs to be done via command instead of kubernetes module
+      #            due to missing flag for --delete-emptydir-data
+      #            ¯\_(ツ)_/¯
+    - name: "Smardigo Patchday: drain node"
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      ansible.builtin.command: "/usr/local/bin/kubectl drain --timeout 2m --ignore-daemonsets --force --delete-emptydir-data {{ inventory_hostname | lower }}"
+      register: node_drained
+      until: node_drained
+      retries: 3
+      delay: 30
+      failed_when: false
+
+    - name: "Smardigo Patchday: stop k8s basic services"
+      ansible.builtin.systemd:
+        name: '{{ item }}'
+        state: stopped
+      loop: '{{ k8s_basic_services }}'
+
+    - name: "Smardigo Patchday: update pkgs"
+      ansible.builtin.apt:
+        upgrade: yes
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+
+    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
+      ansible.builtin.reboot:
+        post_reboot_delay: 30
+        reboot_timeout: 60
+
+    - name: "Smardigo Patchday: wait_for host after reboot"
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        delay: 15
+        timeout: 180
+        port: 22
+        host: '{{ stage_server_ip }}'
+        search_regex: OpenSSH
+
+    - name: "Smardigo Patchday: start k8s basic services"
+      ansible.builtin.systemd:
+        name: '{{ item }}'
+        state: started
+      loop: '{{ k8s_basic_services }}'
+
+    - name: "Smardigo Patchday: wait for node readiness"
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      kubernetes.core.k8s:
+        kind: Node
+        state: present
+        name: '{{ stage_server_ip }}'
+        wait_condition:
+          reason: KubeletReady
+          type: Ready
+          status: True
+        wait_timeout: 120
+      retries: 5
+      delay: 10
+
+    - name: "Smardigo Patchday: uncordon node"
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      kubernetes.core.k8s_drain:
+        state: uncordon
+        name: '{{ inventory_hostname }}'
--- a/roles/elastic/vars/main.yml
+++ b/roles/elastic/vars/main.yml
@ -1,8 +1,4 @@
 ---
-
-elastic_id: "{{ inventory_hostname }}-elastic"
-elastic_exporter_id: "{{ inventory_hostname }}-elastic-exporter"
-
 elastic_docker: {
  networks: [
    {
--- a/roles/kubernetes/base/tasks/main.yml
+++ b/roles/kubernetes/base/tasks/main.yml
@ -20,7 +20,7 @@
  loop:
    - kubernetes
  when:
-  - inventory_hostname == groups['kube_control_plane'][0]
+  - inventory_hostname in groups['kube_control_plane']
  tags:
    - base