diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f518b8a..eb52f83 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ services: stages: - ansible-lint - ansible-builder + - ansible-playbook ansible-lint-job: stage: ansible-lint @@ -37,3 +38,31 @@ ansible-builder-job: tags: - dind - harbor # 05.02.22 TODO some runners run into timeouts + +ansible-patchday-dev: + image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest + stage: ansible-playbook + script: + - export HETZNER_LABEL_SELECTOR='stage=dev' + - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass + - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster + after_script: + - rm /tmp/vault-pass + when: manual + tags: + - dind + - harbor # 05.02.22 TODO some runners run into timeouts + +ansible-patchday-qa: + image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest + stage: ansible-playbook + script: + - export HETZNER_LABEL_SELECTOR='stage=qa' + - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass + - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster + after_script: + - rm /tmp/vault-pass + when: manual + tags: + - dind + - harbor # 05.02.22 TODO some runners run into timeouts diff --git a/ansible.cfg b/ansible.cfg index 500f5a9..8f375c9 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,4 +1,6 @@ [defaults] +pipelining = True +host_key_checking = False inventory_plugins = ./inventory_plugins callbacks_enabled = timer interpreter_python = auto_silent diff --git a/group_vars/all/services.yml b/group_vars/all/services.yml new file mode 100644 index 0000000..07d7109 --- /dev/null +++ b/group_vars/all/services.yml @@ -0,0 +1,4 @@ +--- + +elastic_id: "{{ inventory_hostname }}-elastic" +elastic_exporter_id: "{{ inventory_hostname }}-elastic-exporter" diff --git a/patchday.yml b/patchday.yml new file mode 100644 index 0000000..abbab15 --- /dev/null +++ b/patchday.yml @@ -0,0 +1,218 @@ +--- + +### tags: +### check_elastic_cluster + +- hosts: elastic + serial: 1 + tasks: + - name: "Smardigo Patchday: update pkgs" + ansible.builtin.apt: + upgrade: yes + update_cache: yes + autoremove: yes + autoclean: yes + + - name: "Smardigo Patchday: find docker_compose.yml files" + ansible.builtin.find: + paths: '{{ service_base_path }}' + pattern: 'docker*.yml' + recurse: yes + register: docker_compose_services + + - name: "Smardigo Patchday: shutdown services" + community.docker.docker_compose: + project_src: '{{ item.path | dirname }}' + state: absent + loop: '{{ docker_compose_services.files }}' + + - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" + ansible.builtin.reboot: + post_reboot_delay: 30 + reboot_timeout: 60 + + - name: "Smardigo Patchday: wait_for host after reboot" + delegate_to: localhost + ansible.builtin.wait_for: + delay: 15 + timeout: 180 + port: 22 + host: '{{ stage_server_ip }}' + search_regex: OpenSSH + + - name: "Smardigo Patchday: start services" + community.docker.docker_compose: + project_src: '{{ item.path | dirname }}' + state: present + loop: '{{ docker_compose_services.files }}' + + - name: "Smardigo Patchday: wait until cluster is green" + ansible.builtin.uri: + url: "https://localhost:9200/_cluster/health" + user: "{{ elastic_admin_username }}" + password: "{{ elastic_admin_password }}" + force_basic_auth: true + status_code: 200 + ca_path: "{{ service_base_path }}/{{ elastic_id }}/certs/ca/ca.crt" + register: check_elastic_cluster + delay: 30 + retries: 30 + until: + - check_elastic_cluster.json is defined + - check_elastic_cluster.json.status == 'green' + no_log: true + tags: + - check_elastic_cluster + +- hosts: postgres + serial: 1 + tasks: + - name: "Smardigo Patchday: stop service(s)" + ansible.builtin.systemd: + name: postgresql + state: stopped + + - name: "Smardigo Patchday: update pkgs" + ansible.builtin.apt: + upgrade: yes + update_cache: yes + autoremove: yes + autoclean: yes + + - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" + ansible.builtin.reboot: + post_reboot_delay: 30 + reboot_timeout: 60 + + - name: "Smardigo Patchday: wait_for host after reboot" + delegate_to: localhost + ansible.builtin.wait_for: + delay: 15 + timeout: 180 + port: 22 + host: '{{ stage_server_ip }}' + search_regex: OpenSSH + + - name: "Smardigo Patchday: start services" + ansible.builtin.systemd: + name: postgresql + state: started + +- hosts: all,!elastic,!postgres,!k8s_cluster,!gw + serial: 10 + tasks: + - name: "Smardigo Patchday: update pkgs" + ansible.builtin.apt: + upgrade: yes + update_cache: yes + autoremove: yes + autoclean: yes + + - name: "Smardigo Patchday: find docker_compose.yml files" + ansible.builtin.find: + paths: '{{ service_base_path }}' + pattern: 'docker*.yml' + recurse: yes + register: docker_compose_services + + - name: "Smardigo Patchday: shutdown services" + community.docker.docker_compose: + project_src: '{{ item.path | dirname }}' + state: absent + loop: '{{ docker_compose_services.files }}' + + - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" + ansible.builtin.reboot: + post_reboot_delay: 30 + reboot_timeout: 60 + + - name: "Smardigo Patchday: wait_for host after reboot" + delegate_to: localhost + ansible.builtin.wait_for: + delay: 15 + timeout: 180 + port: 22 + host: '{{ stage_server_ip }}' + search_regex: OpenSSH + + - name: "Smardigo Patchday: start services" + community.docker.docker_compose: + project_src: '{{ item.path | dirname }}' + state: present + loop: '{{ docker_compose_services.files }}' + +- hosts: k8s_cluster + serial: 1 + vars: + k8s_basic_services: + - kubelet + - docker + tasks: + # draining the hard way + # due to force( delete static pods) + relative short terminate_grace_period + + # --delete-local-data to kick pods with emptyDir + # + # ATTENTION: needs to be done via command instead of kubernetes module + # due to missing flag for --delete-emptydir-data + # ¯\_(ツ)_/¯ + - name: "Smardigo Patchday: drain node" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + ansible.builtin.command: "/usr/local/bin/kubectl drain --timeout 2m --ignore-daemonsets --force --delete-emptydir-data {{ inventory_hostname | lower }}" + register: node_drained + until: node_drained + retries: 3 + delay: 30 + failed_when: false + + - name: "Smardigo Patchday: stop k8s basic services" + ansible.builtin.systemd: + name: '{{ item }}' + state: stopped + loop: '{{ k8s_basic_services }}' + + - name: "Smardigo Patchday: update pkgs" + ansible.builtin.apt: + upgrade: yes + update_cache: yes + autoremove: yes + autoclean: yes + + - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>" + ansible.builtin.reboot: + post_reboot_delay: 30 + reboot_timeout: 60 + + - name: "Smardigo Patchday: wait_for host after reboot" + delegate_to: localhost + ansible.builtin.wait_for: + delay: 15 + timeout: 180 + port: 22 + host: '{{ stage_server_ip }}' + search_regex: OpenSSH + + - name: "Smardigo Patchday: start k8s basic services" + ansible.builtin.systemd: + name: '{{ item }}' + state: started + loop: '{{ k8s_basic_services }}' + + - name: "Smardigo Patchday: wait for node readiness" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + kubernetes.core.k8s: + kind: Node + state: present + name: '{{ stage_server_ip }}' + wait_condition: + reason: KubeletReady + type: Ready + status: True + wait_timeout: 120 + retries: 5 + delay: 10 + + - name: "Smardigo Patchday: uncordon node" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + kubernetes.core.k8s_drain: + state: uncordon + name: '{{ inventory_hostname }}' diff --git a/roles/elastic/vars/main.yml b/roles/elastic/vars/main.yml index c4542dc..0f9bc63 100644 --- a/roles/elastic/vars/main.yml +++ b/roles/elastic/vars/main.yml @@ -1,8 +1,4 @@ --- - -elastic_id: "{{ inventory_hostname }}-elastic" -elastic_exporter_id: "{{ inventory_hostname }}-elastic-exporter" - elastic_docker: { networks: [ { diff --git a/roles/kubernetes/base/tasks/main.yml b/roles/kubernetes/base/tasks/main.yml index 7034c7c..a8c6ff1 100644 --- a/roles/kubernetes/base/tasks/main.yml +++ b/roles/kubernetes/base/tasks/main.yml @@ -20,7 +20,7 @@ loop: - kubernetes when: - - inventory_hostname == groups['kube_control_plane'][0] + - inventory_hostname in groups['kube_control_plane'] tags: - base