DEV-273: automate rollout ansible by gitlab

.gitlab-ci.yml

@@ -1,6 +1,8 @@
 ---
 variables:
   AWX_EE_DOCKER_IMAGE_EXTERN: "dev-harbor-01.smardigo.digital/awx/awx-custom-ee"
+  ANSIBLE_HOST_KEY_CHECKING: 'false'
+  ANSIBLE_FORCE_COLOR: 'true'
 
 image: docker.dev-at.de/smardigo/smardigo-ci-ansible
 
@@ -11,7 +13,9 @@ services:
 stages:
   - ansible-lint
   - ansible-builder
-  - ansible-playbook
+  - ansible-run-setup
+  - ansible-run-kubernetes
+  - ansible-patchday
 
 ansible-lint-job:
   stage: ansible-lint
@@ -39,13 +43,193 @@ ansible-builder-job:
     - dind
     - harbor # 05.02.22 TODO some runners run into timeouts
 
-ansible-patchday-dev:
+########
+###   https://patorjk.com/software/taag/#p=display&f=Doom&t=ansible%20-%20run
+###
+###                    _ _     _                                                       _                               _
+###                   (_) |   | |                                                     | |                             | |
+###     __ _ _ __  ___ _| |__ | | ___   ______   _ __ _   _ _ __     ______   ___  ___| |_ _   _ _ __  _   _ _ __ ___ | |
+###    / _` | '_ \/ __| | '_ \| |/ _ \ |______| | '__| | | | '_ \   |______| / __|/ _ \ __| | | | '_ \| | | | '_ ` _ \| |
+###   | (_| | | | \__ \ | |_) | |  __/          | |  | |_| | | | |           \__ \  __/ |_| |_| | |_) | |_| | | | | | | |
+###    \__,_|_| |_|___/_|_.__/|_|\___|          |_|   \__,_|_| |_|           |___/\___|\__|\__,_| .__(_)__, |_| |_| |_|_|
+###                                                                                             | |     __/ |
+###                                                                                             |_|    |___/
+
+
+ansible-run-setup-1-dev:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+    - ssh-add -L
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-run-setup-2-qa:
   image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
-  stage: ansible-playbook
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
   script:
-    - export HETZNER_LABEL_SELECTOR='stage=dev'
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
-    - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass
+    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
-    - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-run-setup-3-prodnso:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
+    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+
+
+
+########
+###   https://patorjk.com/software/taag/#p=display&f=Doom&t=ansible%20-%20run
+###
+###                    _ _     _                                              _          _                          _                             _
+###                   (_) |   | |                                            | |        | |                        | |                           | |
+###     __ _ _ __  ___ _| |__ | | ___   ______   _ __ _   _ _ __     ______  | | ___   _| |__   ___ _ __ _ __   ___| |_ ___  ___  _   _ _ __ ___ | |
+###    / _` | '_ \/ __| | '_ \| |/ _ \ |______| | '__| | | | '_ \   |______| | |/ / | | | '_ \ / _ \ '__| '_ \ / _ \ __/ _ \/ __|| | | | '_ ` _ \| |
+###   | (_| | | | \__ \ | |_) | |  __/          | |  | |_| | | | |           |   <| |_| | |_) |  __/ |  | | | |  __/ ||  __/\__ \| |_| | | | | | | |
+###    \__,_|_| |_|___/_|_.__/|_|\___|          |_|   \__,_|_| |_|           |_|\_\\__,_|_.__/ \___|_|  |_| |_|\___|\__\___||___(_)__, |_| |_| |_|_|
+###                                                                                                                                __/ |
+###                                                                                                                               |___/
+
+ansible-run-kubernetes-1-dev:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+    - ssh-add -L
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-run-kubernetes-2-qa:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-run-kubernetes-3-prodnso:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-run-setup
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
+    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  only:
+    - master
+    - schedules
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+########
+###   https://patorjk.com/software/taag/#p=display&f=Doom&t=patchday
+###                _       _         _
+###               | |     | |       | |
+###    _ __   __ _| |_ ___| |__   __| | __ _ _   _
+###   | '_ \ / _` | __/ __| '_ \ / _` |/ _` | | | |
+###   | |_) | (_| | || (__| | | | (_| | (_| | |_| |
+###   | .__/ \__,_|\__\___|_| |_|\__,_|\__,_|\__, |
+###   | |                                     __/ |
+###   |_|                                    |___/
+###
+
+ansible-patchday-1-dev:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-patchday
+  before_script:
+    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
   when: manual
@@ -53,13 +237,39 @@ ansible-patchday-dev:
     - dind
     - harbor # 05.02.22 TODO some runners run into timeouts
 
-ansible-patchday-qa:
+ansible-patchday-2-qa:
   image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
-  stage: ansible-playbook
+  stage: ansible-patchday
+  before_script:
+    - 'command -v ssh-agent >/qa/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
+  script:
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
+  after_script:
+    - rm /tmp/vault-pass
+  when: manual
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
+ansible-patchday-3-prodnso:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  stage: ansible-patchday
+  before_script:
+    - 'command -v ssh-agent >/qa/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
+    - mkdir -p ~/.ssh
+    - chmod 0700 ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
   script:
-    - export HETZNER_LABEL_SELECTOR='stage=qa'
+    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
-    - echo "${ANSIBLE_VAULT_PASS}" >> /tmp/vault-pass
+    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
-    - ansible-playbook -i stage-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u root -l elastic -t check_elastic_cluster
   after_script:
     - rm /tmp/vault-pass
   when: manual

ansible.cfg

@@ -5,3 +5,4 @@ inventory_plugins = ./inventory_plugins
 callbacks_enabled = timer
 interpreter_python = auto_silent
 log_path=last_ansible_run
+forks = 30

group_vars/all/plain.yml

@@ -101,6 +101,7 @@ default_plattform_users:
   - '{{ admin_user }}'
 
 smardigo_plattform_users:
+  - 'gitlabci' # needed for periodic ansible run
   - 'sven.ketelsen'
   - 'peter.heise'
   - 'claus.paetow'

roles/common/tasks/main.yml

@@ -169,15 +169,6 @@
   tags:
     - install
 
-- name: "Upgrade all packages"
-  apt:
-    update_cache: yes
-    upgrade: yes
-  tags:
-    - install
-    - upgrade
-  when: ansible_distribution == "Ubuntu"
-
 - name: "Ensure docker configuration directory exists"
   file:
     path: '/root/.docker/'

setup.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: 'apply setup to {{ host | default("all") }}'
-  hosts: '{{ host | default("all") }}'
+  hosts: '{{ host | default("all") }},!gw'
   serial: "{{ serial_number | default(10) }}"
   strategy: free
   vars:
@@ -19,15 +19,6 @@
       tags:
         - always
 
-    - name: Updating apt cache
-      apt:
-        update_cache: yes
-      when: ansible_distribution == "Ubuntu"
-      tags:
-        - common
-        - install
-        - upgrade
-
     - name: Remove outdated dependencies
       apt:
         name: [

users/gitlabci/ssh.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC56R1WdRlekdRxDNM2r4pIK+y0t/mzplmZQxXXMxlWVIH8l4Sdqt6K/vteiX8kAepaPmqCo8iV16Dh2a5pxI1TnziJhoY9OmAu75qn/NRj9ePm0ZC/XjULcZS2GhlxlonVb//bvU6uwK4z1cbgsTkp4da39KLd7yyH2mBkCeKpAubp0Tqzve8uf15PUVU6aRF2YLqqhlOut3capISDnp9HTwQxEO6Kxy4qK0gyn18CCl83ThkT1QRJ8xs4vmDCsdv8KH+ia5PwdKX0taQji5oH5iL3in/fAfSTQoutWWPNWMHlUsmkPOml78zGzsRNa9y3CLUFLvQtuBk+o++dn23UAW495juj01i+VE1GsgGUQTjp7X2Ok3X7Ps4vKixLzVx6bTsLdEQCMyQZybAY9OGkGbLO1DpEciIK2NNwmBDZELVKVPcQrZ7TjG55goGP04505YA0XkbdMocO4ko0sAdPvcWF/60hVpORWzbMXl+f0QX5ll/ZsSylpwcDCMKV+zc= gitlabci@git.dev-at.de

users/outdated/gitlabci.ed25519.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGS/ncIBfMt6w3GfYVu6f88+Ig01AOZQlRwNiBPhfCar gitlabci@git.dev-at.de