DEV-655 adjusted filebeat and logstash config

- logstash: json logging
- logstash: json filter skip_on_invalid_json
- filebeat: remove unnecessary fields

.gitlab-ci.yml

@@ -11,16 +11,18 @@ services:
     alias: docker
 
 stages:
-  - ansible-lint
+  - lint
   - ansible-builder
-  - ansible-run-setup
+  - run-setup
-  - ansible-run-kubernetes
+  - run-setup-digitalocean
-  - ansible-patchday
+  - run-kubernetes
+  - run-management-update
+  - run-patchday
 
-ansible-lint-job:
+lint-job:
-  stage: ansible-lint
+  stage: lint
   script:
-    - echo "Running ansible-lint to check for linting violations"
+    - echo "Running lint to check for linting violations"
     - ansible-lint -c ansible-lint.cfg
   only:
     - branches
@@ -29,19 +31,19 @@ ansible-lint-job:
   tags:
     - dind
 
-ansible-builder-job:
+builder-job:
   # A resource group ensures a job is mutually exclusive across different pipelines for the same project.
-  resource_group: deployment
+  resource_group: dev
   stage: ansible-builder
   before_script:
     - cd ansible-builder
   script:
-    - echo "Running ansible-build to build awx execution environment"
+    - echo "Running ansible-builder to build awx execution environment"
     - ansible-builder build -v 3 --tag $AWX_EE_DOCKER_IMAGE_EXTERN:latest
     - docker push $AWX_EE_DOCKER_IMAGE_EXTERN:latest
   only:
     refs:
-      - master
+      - main
     changes:
       - pip-requirements
       - galaxy-requirements.yml
@@ -52,22 +54,30 @@ ansible-builder-job:
     - dind
     - harbor # 05.02.22 TODO some runners run into timeouts
 
+##################################################################################
+
+.run-ansible:
+  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  tags:
+    - dind
+    - harbor # 05.02.22 TODO some runners run into timeouts
+
 ########
-###   https://patorjk.com/software/taag/#p=display&f=Doom&t=ansible%20-%20run
+###   http://patorjk.com/software/taag/#p=display&f=Doom&t=setup.yml
 ###
-###                    _ _     _                                                       _                               _
+###             _                               _ 
-###                   (_) |   | |                                                     | |                             | |
+###            | |                             | |
-###     __ _ _ __  ___ _| |__ | | ___   ______   _ __ _   _ _ __     ______   ___  ___| |_ _   _ _ __  _   _ _ __ ___ | |
+###    ___  ___| |_ _   _ _ __  _   _ _ __ ___ | |
-###    / _` | '_ \/ __| | '_ \| |/ _ \ |______| | '__| | | | '_ \   |______| / __|/ _ \ __| | | | '_ \| | | | '_ ` _ \| |
+###   / __|/ _ \ __| | | | '_ \| | | | '_ ` _ \| |
-###   | (_| | | | \__ \ | |_) | |  __/          | |  | |_| | | | |           \__ \  __/ |_| |_| | |_) | |_| | | | | | | |
+###   \__ \  __/ |_| |_| | |_) | |_| | | | | | | |
-###    \__,_|_| |_|___/_|_.__/|_|\___|          |_|   \__,_|_| |_|           |___/\___|\__|\__,_| .__(_)__, |_| |_| |_|_|
+###   |___/\___|\__|\__,_| .__(_)__, |_| |_| |_|_|
 ###                      | |     __/ |            
 ###                      |_|    |___/             
 
-ansible-run-setup-1-dev:
+.run-setup:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-ansible
-  stage: ansible-run-setup
+  stage: run-setup
-  before_script:
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
@@ -75,81 +85,87 @@ ansible-run-setup-1-dev:
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
     - ssh-add -L
-  script:
+    - export HETZNER_LABEL_SELECTOR="stage=${STAGE}"
-    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --vault-password-file /tmp/vault-pass -t common -u gitlabci
-    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
-  only:
+  except:
-    - master
     - schedules
-  tags:
-    - dind
-    - harbor # 05.02.22 TODO some runners run into timeouts
-  resource_group: dev
 
-ansible-run-setup-2-qa:
+run-setup-digitalocean:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-ansible
-  stage: ansible-run-setup
+  stage: run-setup
   before_script:
+    - export STAGE=dev
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
     - mkdir -p ~/.ssh
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
+    - ssh-add -L
-    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+    - ansible-playbook -i stage-digitalocean setup.yml --vault-password-file /tmp/vault-pass -t common -u gitlabci
-    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
   only:
-    - qa
+    - main
+  except:
     - schedules
-  tags:
+
-    - dind
+run-setup-dev:
-    - harbor # 05.02.22 TODO some runners run into timeouts
+  extends: .run-setup
+  resource_group: dev
+  before_script:
+    - export STAGE=dev
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  only:
+    - main
+
+run-setup-devscr:
+  extends: .run-setup
+  resource_group: devscr
+  before_script:
+    - export STAGE=devscr
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  only:
+    - main
+
+run-setup-qa:
+  extends: .run-setup
   resource_group: qa
+  before_script:
+    - export STAGE=qa
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+  only:
+    - qa
 
-ansible-run-setup-3-prodnso:
+run-setup-prodnso:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-setup
-  stage: ansible-run-setup
+  resource_group: prodnso
   before_script:
-    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - export STAGE=prodnso
-    - eval $(ssh-agent -s)
-    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
-    - mkdir -p ~/.ssh
-    - chmod 0700 ~/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
     - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
-    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml setup.yml --tags common --vault-password-file /tmp/vault-pass -u gitlabci
-  after_script:
-    - rm /tmp/vault-pass
   only:
     - prodnso
-    - schedules
-  tags:
-    - dind
-    - harbor # 05.02.22 TODO some runners run into timeouts
-  resource_group: prodnso
 
 ########
-###   https://patorjk.com/software/taag/#p=display&f=Doom&t=ansible%20-%20run
+###   This Page: http://patorjk.com/software/taag/#p=display&f=Doom&t=kubernetes.yml
 ###
-###                    _ _     _                                              _          _                          _                             _
+###    _          _                          _                             _
-###                   (_) |   | |                                            | |        | |                        | |                           | |
+###   | |        | |                        | |                           | |
-###     __ _ _ __  ___ _| |__ | | ___   ______   _ __ _   _ _ __     ______  | | ___   _| |__   ___ _ __ _ __   ___| |_ ___  ___  _   _ _ __ ___ | |
+###   | | ___   _| |__   ___ _ __ _ __   ___| |_ ___  ___  _   _ _ __ ___ | |
-###    / _` | '_ \/ __| | '_ \| |/ _ \ |______| | '__| | | | '_ \   |______| | |/ / | | | '_ \ / _ \ '__| '_ \ / _ \ __/ _ \/ __|| | | | '_ ` _ \| |
+###   | |/ / | | | '_ \ / _ \ '__| '_ \ / _ \ __/ _ \/ __|| | | | '_ ` _ \| |
-###   | (_| | | | \__ \ | |_) | |  __/          | |  | |_| | | | |           |   <| |_| | |_) |  __/ |  | | | |  __/ ||  __/\__ \| |_| | | | | | | |
+###   |   <| |_| | |_) |  __/ |  | | | |  __/ ||  __/\__ \| |_| | | | | | | |
-###    \__,_|_| |_|___/_|_.__/|_|\___|          |_|   \__,_|_| |_|           |_|\_\\__,_|_.__/ \___|_|  |_| |_|\___|\__\___||___(_)__, |_| |_| |_|_|
+###   |_|\_\\__,_|_.__/ \___|_|  |_| |_|\___|\__\___||___(_)__, |_| |_| |_|_|
 ###                                                         __/ |
 ###                                                        |___/
 
-ansible-run-kubernetes-1-dev:
+.run-kubernetes:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-ansible
-  stage: ansible-run-kubernetes
+  stage: run-kubernetes
-  before_script:
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
@@ -157,144 +173,172 @@ ansible-run-kubernetes-1-dev:
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
     - ssh-add -L
-  script:
+    - export HETZNER_LABEL_SELECTOR="stage=${STAGE}"
-    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
-    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
-  only:
+  except:
-    - master
     - schedules
-  tags:
+
-    - dind
+run-kubernetes-dev:
-    - harbor # 05.02.22 TODO some runners run into timeouts
+  extends: .run-kubernetes
   resource_group: dev
+  before_script:
+    - export STAGE=dev
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  only:
+    - main
 
-ansible-run-kubernetes-2-qa:
+run-kubernetes-qa:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-kubernetes
-  stage: ansible-run-kubernetes
+  resource_group: qa
   before_script:
-    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - export STAGE=qa
-    - eval $(ssh-agent -s)
-    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
-    - mkdir -p ~/.ssh
-    - chmod 0700 ~/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
     - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
-    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
-  after_script:
-    - rm /tmp/vault-pass
   only:
     - qa
-    - schedules
-  tags:
-    - dind
-    - harbor # 05.02.22 TODO some runners run into timeouts
-  resource_group: qa
 
-ansible-run-kubernetes-3-prodnso:
+run-kubernetes-prodnso:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-kubernetes
-  stage: ansible-run-kubernetes
+  resource_group: prodnso
   before_script:
-    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
+    - export STAGE=prodnso
-    - eval $(ssh-agent -s)
-    - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
-    - mkdir -p ~/.ssh
-    - chmod 0700 ~/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
     - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
-    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml kubernetes.yml --vault-password-file /tmp/vault-pass -u gitlabci
-  after_script:
-    - rm /tmp/vault-pass
   only:
     - prodnso
-    - schedules
-  tags:
-    - dind
-    - harbor # 05.02.22 TODO some runners run into timeouts
-  resource_group: prodnso
 
 ########
-###   https://patorjk.com/software/taag/#p=display&f=Doom&t=patchday
+###   http://patorjk.com/software/taag/#p=display&f=Doom&t=smardigo.yml
-###                _       _         _
-###               | |     | |       | |
-###    _ __   __ _| |_ ___| |__   __| | __ _ _   _
-###   | '_ \ / _` | __/ __| '_ \ / _` |/ _` | | | |
-###   | |_) | (_| | || (__| | | | (_| | (_| | |_| |
-###   | .__/ \__,_|\__\___|_| |_|\__,_|\__,_|\__, |
-###   | |                                     __/ |
-###   |_|                                    |___/
 ###
+###                                _ _                             _ 
+###                               | (_)                           | |
+###    ___ _ __ ___   __ _ _ __ __| |_  __ _  ___  _   _ _ __ ___ | |
+###   / __| '_ ` _ \ / _` | '__/ _` | |/ _` |/ _ \| | | | '_ ` _ \| |
+###   \__ \ | | | | | (_| | | | (_| | | (_| | (_) | |_| | | | | | | |
+###   |___/_| |_| |_|\__,_|_|  \__,_|_|\__, |\___(_)__, |_| |_| |_|_|
+###                                     __/ |       __/ |            
+###                                    |___/       |___/             
 
-ansible-patchday-1-dev:
+.run-management-update:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-ansible
-  stage: ansible-patchday
+  stage: run-management-update
-  before_script:
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
     - mkdir -p ~/.ssh
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
+    - ssh-add -L
-    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+    - export HETZNER_LABEL_SELECTOR="stage=${STAGE}"
-    - STAGE=dev && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
+    - ansible-playbook -i stage-$STAGE smardigo.yml --vault-password-file=/tmp/vault-pass -l management -t update_configurations -u gitlabci
   after_script:
     - rm /tmp/vault-pass
-  when: manual
   only:
-    - master
+    changes:
-  tags:
+      - smardigo/**/*
-    - dind
+  except:
-    - harbor # 05.02.22 TODO some runners run into timeouts
+    - schedules
+
+run-management-update-dev:
+  extends: .run-management-update
   resource_group: dev
+  before_script:
+    - export STAGE=dev
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  only:
+    - main
 
-ansible-patchday-2-qa:
+run-management-update-qa:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-management-update
-  stage: ansible-patchday
+  resource_group: qa
   before_script:
+    - export STAGE=qa
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+  only:
+    - qa
+
+run-management-update-prodnso:
+  extends: .run-management-update
+  resource_group: prodnso
+  before_script:
+    - export STAGE=prodnso
+    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
+  only:
+    - prodnso
+
+########
+###   http://patorjk.com/software/taag/#p=display&f=Doom&t=patchday.yml
+###   
+###                _       _         _                              _ 
+###               | |     | |       | |                            | |
+###    _ __   __ _| |_ ___| |__   __| | __ _ _   _  _   _ _ __ ___ | |
+###   | '_ \ / _` | __/ __| '_ \ / _` |/ _` | | | || | | | '_ ` _ \| |
+###   | |_) | (_| | || (__| | | | (_| | (_| | |_| || |_| | | | | | | |
+###   | .__/ \__,_|\__\___|_| |_|\__,_|\__,_|\__, (_)__, |_| |_| |_|_|
+###   | |                                     __/ |  __/ |            
+###   |_|                                    |___/  |___/             
+
+.run-patchday:
+  extends: .run-ansible
+  stage: run-patchday
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
     - mkdir -p ~/.ssh
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
+    - ssh-add -L
-    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+    - export HETZNER_LABEL_SELECTOR="stage=${STAGE}"
-    - STAGE=qa && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
+    - ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
-  when: manual
+  timeout: 2h
-  only:
-    - qa
-    - schedules
-  tags:
-    - dind
-    - harbor # 05.02.22 TODO some runners run into timeouts
-  resource_group: qa
 
-ansible-patchday-3-prodnso:
+run-patchday-dev:
-  image: $AWX_EE_DOCKER_IMAGE_EXTERN:latest
+  extends: .run-patchday
-  stage: ansible-patchday
+  resource_group: dev
   before_script:
+    - export STAGE=dev
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"
+
+run-patchday-dev-digitalocean:
+  extends: .run-ansible
+  stage: run-patchday
+  before_script:
+    - echo "${ANSIBLE_VAULT_PASS_DEV}" > /tmp/vault-pass
+  script:
     - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
     - 'echo "$GITLAB_SSH_KEY" | tr -d "\r" | ssh-add -'
     - mkdir -p ~/.ssh
     - chmod 0700 ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config'
-  script:
+    - ssh-add -L
-    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
+    - ansible-playbook -i stage-digitalocean patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
-    - STAGE=prodnso && HETZNER_LABEL_SELECTOR="stage=${STAGE}" && ansible-playbook -i stage-${STAGE}-netgo-hcloud.yml patchday.yml --vault-password-file=/tmp/vault-pass -u gitlabci
   after_script:
     - rm /tmp/vault-pass
-  when: manual
+  timeout: 2h
-  only:
+  rules:
-    - prodnso
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"
-    - schedules
+
-  tags:
+run-patchday-qa:
-    - dind
+  extends: .run-patchday
-    - harbor # 05.02.22 TODO some runners run into timeouts
+  resource_group: qa
+  before_script:
+    - export STAGE=qa
+    - echo "${ANSIBLE_VAULT_PASS_QA}" > /tmp/vault-pass
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "qa"
+
+run-patchday-prodnso:
+  extends: .run-patchday
   resource_group: prodnso
+  before_script:
+    - export STAGE=prodnso
+    - echo "${ANSIBLE_VAULT_PASS_PRODNSO}" > /tmp/vault-pass
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "prodnso"

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "kubespray"]
   path = kubespray
   url = https://github.com/kubernetes-sigs/kubespray.git
-  branch = v2.18.0
+  branch = release-2.19

README.md

@@ -91,10 +91,6 @@ if everything works fine, plz push the created docker container with:
 
 # TODO
 
-IPFire
-  149.233.6.129 - eShelter
-  212.121.131.106 - Siemensdamm
-
 Prometheus (Grafana)
   docker exec -i dev-prometheus-01-grafana sh -c 'grafana-cli plugins install grafana-piechart-panel'
   docker restart dev-prometheus-01-grafana

ansible.cfg

@@ -2,7 +2,7 @@
 pipelining = True
 host_key_checking = False
 inventory_plugins = ./inventory_plugins
-callbacks_enabled = timer
+callbacks_enabled = profile_tasks
 interpreter_python = auto_silent
 log_path=last_ansible_run
 forks = 30

create-database-backup.yml

@@ -54,6 +54,16 @@
       with_items: "{{ cluster_features }}"
       when: item in ['connect', 'management_connect', 'keycloak', 'webdav', 'gitea', 'workflow_index', 'workflow_proxy', 'pdns']
 
+    - name: "Add maria servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-maria-01"
+        groups:
+        - "stage_{{ stage }}"
+        - "{{ item }}"
+      changed_when: False
+      with_items: "{{ cluster_features }}"
+      when: item in ['connect_wordpress']
+
 #############################################################
 #   Creating database backups for created inventory
 #############################################################
@@ -62,7 +72,7 @@
   serial: "{{ serial_number | default(1) }}"
   remote_user: root
   vars:
-    postgres_backup_state: dump
+    database_backup_state: dump
     ansible_ssh_host: "{{ stage_server_domain }}"
 
   roles:
@@ -75,6 +85,12 @@
     - role: keycloak_postgres
       when: "'keycloak' in group_names"
 
+#    - role: pdns_admin_postgres
+#      when: "'pdns' in group_names"
+
+#    - role: pdns_postgres
+#      when: "'pdns' in group_names"
+
     - role: webdav_postgres
       when: "'webdav' in group_names"
 
@@ -84,6 +100,9 @@
     - role: workflow_proxy_postgres
       when: "'workflow_proxy' in group_names"
 
+    - role: connect_wordpress_maria
+      when: "'connect_wordpress' in group_names"
+
 #############################################################
 #   Sending smardigo management message to process
 #############################################################

create-database.yml

@@ -58,7 +58,7 @@
         - "{{ item }}"
       changed_when: False
       with_items: "{{ cluster_features }}"
-      when: item in ['connect', 'management_connect', 'keycloak', 'webdav', 'gitea', 'workflow_index', 'workflow_proxy', 'pdns']
+      when: item in ['confirm', 'connect', 'management_connect', 'keycloak', 'webdav', 'gitea', 'workflow_index', 'workflow_proxy', 'pdns']
 
     - name: "Add maria servers to hosts if necessary"
       add_host:
@@ -88,9 +88,18 @@
         - always
 
   roles:
+    - role: confirm_postgres
+      when: "'confirm' in group_names"
+
     - role: connect_postgres
       when: "'connect' in group_names"
 
+    - role: gitea_postgres
+      when: "'gitea' in group_names"
+
+    - role: keycloak_postgres
+      when: "'keycloak' in group_names"
+
     - role: pdns_postgres
       vars:
         initialize: True
@@ -101,12 +110,6 @@
         initialize: True
       when: "'pdns' in group_names"
 
-    - role: gitea_postgres
-      when: "'gitea' in group_names"
-
-    - role: keycloak_postgres
-      when: "'keycloak' in group_names"
-
     - role: webdav_postgres
       when: "'webdav' in group_names"
 

create-kibana-objects.yml

@@ -61,7 +61,7 @@
   vars:
     ansible_connection: local
     ansible_ssh_host: "{{ stage_server_domain }}"
-    api_endpoint: '{{ stage }}-elastic-stack-kibana-01-kibana.{{ domain }}'
+    kibana_api_endpoint: '{{ shared_service_elastic_stack_kibana_01_hostname }}-kibana.{{ domain }}'
     elastic_state: present
     elastic_users:
       -

create-remote-database-backup.yml

@@ -11,6 +11,7 @@
 # Parameters:
 #   playbook inventory
 #     stage := the name of the stage (e.g. dev, int, qa, prod)
+#     database_engine := the database engine to generate a complete backup for (e.g. postgres, maria)
 #   smardigo message callback
 #     scope_id := (scope id of the management process)
 #     process_instance_id := (process instance id of the management process)
@@ -50,7 +51,7 @@
       changed_when: False
     - name: "Add 'storage' servers to hosts if necessary"
       add_host:
-        name: "{{ stage }}-fgrz-01"
+        name: "{{ stage }}-backup-01"
         groups:
         - "stage_{{ stage }}"
         - storage
@@ -62,6 +63,11 @@
 
 - hosts: "postgres:maria"
   serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+    current_date_time: '{{ get_current_date_time }}'
+
   tasks:
     - name: "Trigger backup mechanism"
       include_role:
@@ -74,8 +80,11 @@
 
 - hosts: "postgres:maria:storage"
   serial: "{{ serial_number | default(5) }}"
+  gather_facts: false
   vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
     storageserver_system_user: 'backuphamster'
+
   tasks:
     # I could not get it up and running with <synchronize> module
     # to sync data from remote server A to remote server B
@@ -83,7 +92,8 @@
       become: yes
       become_user: '{{ storageserver_system_user }}'
       vars:
-        database_server_ip: "{{ stage }}-{{ database_engine }}-01.{{ domain }}"
+        # should work with non-fqdn due to existing entry in /etc/hosts
+        database_server_ip: "{{ stage }}-{{ database_engine }}-01"
       shell: '/home/{{ storageserver_system_user }}/pull_remote_backups.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
       when:
         - inventory_hostname in groups['storage']
@@ -91,7 +101,7 @@
     - name: "Cleanup remote backup dirs: {{ database_engine }}"
       become: yes
       file:
-        path: '{{ backup_directory }}/{{ database_engine }}/{{ ansible_date_time.date }}'
+        path: '{{ backup_directory }}/{{ database_engine }}/{{ get_current_date }}'
         state: absent
       when:
         - not inventory_hostname in groups['storage']

create-server.yml

@@ -43,6 +43,7 @@
         groups:
         - "stage_{{ stage }}"
         - "{{ cluster_service }}"
+        - hcloud
       with_sequence: start=1 end={{ cluster_size | default(1) }}
       changed_when: False
 
@@ -52,6 +53,7 @@
 
 - hosts: "stage_{{ stage }}:!{{ stage }}-virtual-host-to-read-groups-vars"
   serial: "{{ serial_number | default(5) }}"
+  remote_user: root
   gather_facts: false
 
   pre_tasks:

elastic-certs.sh

@@ -1,3 +1,9 @@
 #!/bin/bash
 
+if [ "x$1" == "x" ];then
+  echo "Stage as param \$1 is missing. exit"
+  exit 1
+fi
+
+
 docker run -v `pwd`/templates/elastic-certs:/certs -v `pwd`/templates/elastic-certs/$1-instances.yaml:/usr/share/elasticsearch/config/certificates/$1-instances.yml docker.elastic.co/elasticsearch/elasticsearch:7.12.0 /bin/sh "/certs/certutil.sh" $1

evil-remove-server.yml

@@ -42,7 +42,7 @@
           - server_state: "absent"
       - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
         include_role:
-          name: sma_digitalocean
+          name: dns
           tasks_from: _remove_dns
         vars:
           record_to_remove: '{{ inventory_hostname }}'

export-database.yml

@@ -0,0 +1,99 @@
+---
+
+# Parameters:
+#   playbook inventory
+#     stage := the name of the stage (e.g. dev, int, qa, prod)
+#     tenant_id := (unique key for the tenant, e.g. customer)
+#     cluster_name := (business name for the cluster, e.g. product, department )
+#     cluster_size := (WIP node count for the cluster)
+#     cluster_service := (service to setup, e.g. 'connect', ...)
+#     cluster_features := (optional features to use, e.g. ['wordpress', 'resubmission', ...])
+#     database_backup_file := the dump file to export, has to be on the database server under /tmp (e.g. wordpress_portal.sql)
+#     target_database := (optional) the database to export into ( see {{ connect_wordpress_maria_database }})
+#   smardigo message callback
+#     scope_id := (scope id of the management process)
+#     process_instance_id := (process instance id of the management process)
+#     smardigo_management_action := (smardigo management action anme of the management process)
+
+#############################################################
+#   Creating inventory dynamically for given parameters
+#############################################################
+
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+
+#     add virtual server to load stage specific variables as context
+    - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
+      add_host:
+        name: "{{ stage }}-virtual-host-to-read-groups-vars"
+        groups:
+        - "stage_{{ stage }}"
+      changed_when: False
+
+  tasks:
+    - name: Add maria servers to hosts if necessary
+      add_host:
+        name: "{{ stage }}-maria-01"
+        groups:
+        - "stage_{{ stage }}"
+        - "{{ item }}"
+      changed_when: False
+      with_items: "{{ cluster_features }}"
+      when: item in ['connect_wordpress']
+
+#############################################################
+#   exporting database backups for created inventory
+#############################################################
+
+- hosts: "stage_{{ stage }}:!{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  remote_user: root
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+
+  pre_tasks:
+    - name: "export autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      become: false
+      tags:
+        - always
+
+  roles:
+    - role: export_maria_database
+      vars:
+        database_backup_file: "{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-wordpress.sql.gz"
+      when:
+        - "'connect_wordpress' in group_names"
+        - "target_database is defined"
+
+    - role: export_maria_database
+      vars:
+        target_database: "{{ connect_wordpress_maria_database }}"
+        database_backup_file: "{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-wordpress.sql.gz"
+      when:
+        - "'connect_wordpress' in group_names"
+
+#############################################################
+#   Sending smardigo management message to process
+#############################################################
+
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  run_once: true
+  vars:
+    connect_jwt_username: "{{ management_admin_username }}"
+
+  tasks:
+    - name: "Sending smardigo management message to <{{ smardigo_management_url }}>"
+      include_tasks: tasks/smardigo_management_message.yml

external_monitoring.yml

@@ -46,10 +46,6 @@
     tags:
       - ssh_hardening
 
-  - name: "Install node-exporter via include_role"
-    include_role:
-      name:  cloudalchemy.node-exporter
-
   - name: "Install blackbox-exporter via include_role"
     include_role:
       name: cloudalchemy.blackbox-exporter

galaxy-requirements.yml

@@ -20,6 +20,8 @@ roles:
   version: v3.6.1
   src: https://github.com/Oefenweb/ansible-postfix.git
   scm: git
+- name: geerlingguy.mysql
+  version: 3.3.2
 
 collections:
 - name: hetzner.hcloud
@@ -35,3 +37,5 @@ collections:
 - name: devsec.hardening
   version: 7.12.0
   src: https://github.com/dev-sec/ansible-collection-hardening
+- name: community.dns
+  version: 2.3.4

gitlab-mirrors.yml

@@ -0,0 +1,60 @@
+---
+# Parameters:
+#   playbook inventory
+#     stage := the name of the stage (e.g. dev, int, qa, prod)
+#   environment variable
+#     GITLAB_API_TOKEN := Access token from gitlab
+
+#############################################################
+#   Creating inventory dynamically for given parameters
+#############################################################
+
+- hosts: localhost
+  gather_facts: false
+  connection: local
+
+  tasks:
+    - name: Add hosts
+      add_host:
+        name: "{{ stage }}-gitlab"
+        groups: "{{ ['stage_' + stage ] }}"
+
+#############################################################
+#   Creating gitlab mirrors for current stage
+#############################################################
+
+- hosts: "stage_{{ stage }}"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  vars:
+    projects:
+    - id: 1210
+      name: argocd
+    - id: 1216
+      name: operator-awx
+    - id: 1212
+      name: operator-jaeger
+    - id: 1231
+      name: operator-knative
+    - id: 1233
+      name: smardigo-awx
+    - id: 1232
+      name: smardigo-jaeger
+
+  pre_tasks:
+    - name: "Add repository remote mirror to project"
+      delegate_to: 127.0.0.1
+      become: false
+      uri:
+        url: "https://git.dev-at.de/api/v4/projects/{{ item.id }}/remote_mirrors"
+        method: POST
+        body_format: json
+        body:
+          enabled: true
+          only_protected_branches: true
+          url: "https://{{ gitea_admin_username }}:{{ gitea_admin_password }}@{{ shared_service_gitea_hostname }}/argocd/{{ item.name }}.git"
+        headers:
+          PRIVATE-TOKEN: "{{ lookup('env', 'GITLAB_API_TOKEN') }}"
+        status_code: [201]
+      loop: "{{ projects }}"

group_vars/all/connect.yml

@@ -0,0 +1,4 @@
+---
+
+connect_client_admin_username: "connect-admin"
+connect_realm_admin_username: "connect-realm-admin"

group_vars/all/dns.yml

@@ -0,0 +1,8 @@
+---
+
+dns: digitalocean
+domain: "smardigo.digital"
+traefik_letsencrypt_provider: "digitalocean"
+
+hetzner_dns_api_key: '{{ hetzner_dns_api_key_vault }}'
+digitalocean_authentication_token: '{{ digitalocean_authentication_token_vault }}'

group_vars/all/firewall.yml

@@ -0,0 +1,269 @@
+---
+hcloud_firewall_objects:
+   -
+     name: "{{ stage }}-default"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: icmp
+        port: ''
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: ICMP allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '22'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: SSH allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '80'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: HTTP allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: HTTPS allowed
+     -
+        direction: in
+        protocol: tcp
+        port: 'any'
+        source_ips: '{{ ip_whitelist_admins }}'
+        destination_ips: []
+        description: TCP - allow work from home without VPN
+     -
+        direction: in
+        protocol: udp
+        port: 'any'
+        source_ips: '{{ ip_whitelist_admins }}'
+        destination_ips: []
+        description: UDP - allow work from home without VPN
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }}'
+   -
+     name: "{{ stage }}-monitoring"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '9080-9085'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: 'Server/Service Monitoring'
+     -
+        direction: in
+        protocol: tcp
+        port: '9001'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: 'PgAdmin'
+     -
+        direction: in
+        protocol: tcp
+        port: '9187'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: 'Postgres-Exporter'
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }}'
+   -
+     name: "{{ stage }}-monitoring-extern-https"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips:
+         - "{{ lookup('community.general.dig', 'dev-blackbox-01.smardigo.digital' ) }}/32"
+        destination_ips: []
+        description: null
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=connect'
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=keycloak'
+
+hcloud_firewall_objects_awx:
+   -
+     name: "{{ stage }}-awx-ssh-access-for-k8s-nodes"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '22'
+        source_ips: "{{ awx_source_ips }}"
+        destination_ips: []
+        description: null
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=awx'
+
+hcloud_firewall_objects_backup:
+   -
+     name: "{{ stage }}-backup-ssh-access"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '22'
+        source_ips:
+          - "{{ offsite_storage_server_ip }}"
+        destination_ips: []
+        description: null
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=backup'
+
+hcloud_firewall_objects_gitea:
+   -
+     name: "{{ stage }}-access-to-gitea"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ ip_whitelist }}"
+        destination_ips: []
+        description: "Allow access for whitelisted ips"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + awx_source_ips }}"
+        destination_ips: []
+        description: "Allow access for kubernetes worker nodes"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + (gitea_https_whitelisted_ips | default([])) }}"
+        destination_ips: []
+        description: "Allow access for custom whitelisted ips"
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=gitea'
+
+hcloud_firewall_objects_keycloak:
+   -
+     name: "{{ stage }}-access-to-keycloak"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ ip_whitelist }}"
+        destination_ips: []
+        description: "Allow access for whitelisted ips"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + awx_source_ips }}"
+        destination_ips: []
+        description: "Allow access for kubernetes worker nodes"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + (keycloak_https_whitelisted_ips | default([])) }}"
+        destination_ips: []
+        description: "Allow access for custom whitelisted ips"
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=keycloak'
+
+hcloud_firewall_objects_kibana:
+   -
+     name: "{{ stage }}-access-to-kibana"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ ip_whitelist }}"
+        destination_ips: []
+        description: "Allow access for whitelisted ips"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + awx_source_ips }}"
+        destination_ips: []
+        description: "Allow access for kubernetes worker nodes"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + (kibana_https_whitelisted_ips | default([])) }}"
+        destination_ips: []
+        description: "Allow access for custom whitelisted ips"
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=kibana'
+
+hcloud_firewall_objects_management:
+   -
+     name: "{{ stage }}-access-to-management"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ ip_whitelist }}"
+        destination_ips: []
+        description: "Allow access for whitelisted ips"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + awx_source_ips }}"
+        destination_ips: []
+        description: "Allow access for kubernetes worker nodes"
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: "{{ [shared_service_network] + (management_https_whitelisted_ips | default([])) }}"
+        destination_ips: []
+        description: "Allow access for custom whitelisted ips"
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }},service=connect,tenant=management'

group_vars/all/plain.yml

@@ -39,17 +39,19 @@ common_apt_dependencies:
   - zip
   - curl
   - htop
+  - iotop
   - net-tools
   - bash-completion
   - python3-pip
 
 common_pip_dependencies:
   - docker-compose
+  - requests>=2.28.1
+  - passlib
 
 use_ssl: true
 http_s: "http{{ use_ssl | ternary('s', '', omit) }}"
 
-domain: "smardigo.digital"
 stage_server_domain: "{{ inventory_hostname }}.{{ domain }}"
 stage_server_url: "{{ http_s }}://{{ stage_server_domain }}"
 
@@ -62,16 +64,24 @@ awx_ansible_user_name: "awx"
 awx_ansible_user_ssh_key_private: "{{ ansible_ssh_key_private_vault }}"
 awx_credential_machine_hetzner_name: hetzner-ansible-ssh
 
+awx_ansible_username: ansible
+awx_ansible_password: ansible
+
+argocd_bootstrap_infrastructure: false
+
 gitlab_ansible_user_name: "gitlabci"
 
+backupuser_user_name: backupuser
+
 # used for root-access by hetzner on server creation (@see cloud console/security/ssh-keys)
 hetzner_ssh_keys:
   - "claus.paetow@netgo.de"
   - "friedrich.goerz@netgo.de"
-  - "peter.heise@netgo.de"
   - "sven.ketelsen@netgo.de"
+  - "michael.haehnel@netgo.de"
+  - "hoan.to@netgo.de"
   - "{{ awx_ansible_user_name }}@netgo.de"
-  - "{{ gitlab_ansible_user_name }}@netgo.de"
+  - "{{ gitlab_ansible_user_name }}@git.dev-at.de"
 
 hetzner_server_labels: "stage={{ stage }}"
 
@@ -99,25 +109,26 @@ sudo_group: "{{ sudo_groups
   | replace('.','-') }}"
 
 # whitelist for outdated user detection - they wont't be deleted at all
-default_plattform_users:
+default_users:
   - 'nobody'
   - 'elastic'
   - 'postgres'
   - 'administrator'
   - '{{ admin_user }}'
-  - '{{ backupuser_username }}'
 
-smardigo_plattform_users:
+default_plattform_users:
   - 'claus.paetow'
   - 'friedrich.goerz'
-  - 'peter.heise'
   - 'sven.ketelsen'
+  - 'michael.haehnel'
+  - 'hoan.to'
   - '{{ awx_ansible_user_name }}'
   - '{{ gitlab_ansible_user_name }}'
 
+smardigo_plattform_users: "{{ default_plattform_users + custom_plattform_users | default([]) }}"
+
 ip_whitelist_admins:
-  - "79.215.10.239/32" # sven
+  - "79.215.12.94/32" # sven
-  - "212.86.56.112/32" # peter
 
 ip_whitelist:
   - "212.121.131.106/32" # netgo berlin
@@ -125,9 +136,7 @@ ip_whitelist:
   - "46.245.219.98/32" # netgo borken
   - "{{ shared_service_network }}"
 
-# for test purpose DEV-361
+offsite_storage_server_ip: 142.132.155.83/32
-# currently (2022.03.18) set to IP of hetzner VM
-gitlab_storage_server: 167.235.18.147/32
 
 docker_owner: "{{ admin_user }}"
 docker_group: "{{ admin_user }}"
@@ -137,12 +146,14 @@ docker_compose_path: "/usr/bin/docker-compose"
 
 service_base_path: '/etc/smardigo'
 
-gitea_admin_email: "nso.devops@netgo.de"
+devops_email_address: "nso.devops@netgo.de"
-lets_encrypt_email: "nso.devops@netgo.de"
+gitea_admin_email: '{{ devops_email_address }}'
-connect_admin_email: "nso.devops@netgo.de"
+lets_encrypt_email: '{{ devops_email_address }}'
-keycloak_admin_email: "nso.devops@netgo.de"
+connect_admin_email: '{{ devops_email_address }}'
-pgadmin4_admin_email: "nso.devops@netgo.de"
+keycloak_admin_email: '{{ devops_email_address }}'
-harbor_oidc_admin_email: "nso.devops@netgo.de"
+pgadmin4_admin_email: '{{ devops_email_address }}'
+harbor_oidc_admin_email: '{{ devops_email_address }}'
+grafana_admin_email: '{{ devops_email_address }}'
 
 http_port: "80"
 https_port: "443"
@@ -191,127 +202,15 @@ backup_directory: "/backups"
 blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
 blackbox_http_2xx_targets:
 - 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
+- 'https://{{ stage }}-kube-awx.smardigo.digital'
 #- 'https://{{ stage }}-management-01-connect.smardigo.digital/'
 blackbox_http_2xx_additional_targets: []
 
 prometheus_federation_enabled: true
 kubernetes_prometheus_endpoint: "{{ stage }}-kube-prometheus.{{ domain }}"
 
-backupuser_username: backupuser
+get_current_date: "{{ lookup('pipe','date +%Y-%m-%d') }}"
-backupuser_ssh_pubkey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDAFRYAy3PqimYUWcO4Q9pdTvDQTsq7hKjWYoQEsJICnRRv+W+5d2lJvC3gqMpmWy9XxtrYePkVHCgIvfJSas9Jv7n7eeYoeWLWJq0nRSKg6EKFCH9y3v8tGPJQQf7wogOhHwr6m79c+lpNVUsVR+QOf76+47ZuwnuEBzK6xbDkmwyt7SPrJ59IFxOlmtz2HgVlTLczLalMygM4qlXqIt+lwuuFz4CsGcr4TwMKp9Uk6SCP3OV12oLnUUUOA3r72qmE4+JeUN6VNbXoBXEANfXm5kbM8w+dFhulCi1fQZCssB8PStA7Cs0gVqL6DYNUKRZaFL8e77hljGkPlOQDxOsBexPuceSDmmr6s5qT1wA6bnEFoeWbLlxixGlFA+1Q/LqWsYzoOZiTHDoaXvsc4VizlPp4Fn0OgJefPjuzBsWOyf0ob5oucfnmCAvEh/k+ioq0bIQDcliAM1UezitblHQgGHhqnKPMi664i0ULLiExARe4IV3KJiaG++RJyzUL5HNz3Qru+K5/pdj2jffluYTC4w+6ZYfjWEZS/DAumExv9T97kFOsapHCQJwTBa368Ch6uKkPCZO8p/ra3xTIUh/PibHaVCadgX2NR9q6jdiQtmc0SOyNJlMlPZD/Q1NrjXJ18ASny7gCBFItMyMtinVx9xQxQ+PFLB8oNYERw1ejIw== storage-server-smardigo'
+get_current_date_time: "{{ lookup('pipe','date +%Y-%m-%d_%H:%M') }}"
-
-current_date_time: "{{ lookup('pipe','date +%Y-%m-%d_%H:%M') }}"
-
-hcloud_firewall_objects:
-   -
-     name: "{{ stage }}-default"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: icmp
-        port: ''
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: ICMP allowed
-     -
-        direction: in
-        protocol: tcp
-        port: '22'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: SSH allowed
-     -
-        direction: in
-        protocol: tcp
-        port: '80'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: HTTP allowed
-     -
-        direction: in
-        protocol: tcp
-        port: '443'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: HTTPS allowed
-     -
-        direction: in
-        protocol: tcp
-        port: 'any'
-        source_ips: '{{ ip_whitelist_admins }}'
-        destination_ips: []
-        description: TCP - allow work from home without VPN
-     -
-        direction: in
-        protocol: udp
-        port: 'any'
-        source_ips: '{{ ip_whitelist_admins }}'
-        destination_ips: []
-        description: UDP - allow work from home without VPN
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'stage={{ stage }}'
-   -
-     name: "{{ stage }}-monitoring"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: tcp
-        port: '9080-9085'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: 'Server/Service Monitoring'
-     -
-        direction: in
-        protocol: tcp
-        port: '9001'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: 'PgAdmin'
-     -
-        direction: in
-        protocol: tcp
-        port: '9187'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: 'Postgres-Exporter'
-     -
-        direction: in
-        protocol: tcp
-        port: '80'
-        source_ips: '{{ ip_whitelist + ip_whitelist_admins }}'
-        destination_ips: []
-        description: 'AWX'
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'stage={{ stage }}'
-   -
-     name: "{{ stage }}-monitoring-extern-https"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: tcp
-        port: '443'
-        source_ips:
-         - "{{ lookup('community.general.dig', 'dev-blackbox-01.smardigo.digital' ) }}/32"
-        destination_ips: []
-        description: null
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=connect'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=keycloak'
 
 hetzner_authentication_ansible: "{{ hetzner_authentication_ansible_vault }}"
 hetzner_authentication_ccm: "{{ hetzner_authentication_ccm_vault }}"
@@ -321,83 +220,14 @@ k8s_basic_services:
   - kubelet
   - containerd
 
-hcloud_firewall_objects_awx:
+selfsigned_ca_private_key_passphrase: '{{ selfsigned_ca_private_key_passphrase_vault }}'
-   -
+  
-     name: "{{ stage }}-awx-ssh-access-for-k8s-nodes"
+prometheus_alert_diskspaceusage_warning: 85
-     state: present
+prometheus_alert_pg_replication_lag: 120
-     rules:
+
-     -
+# hetzner upstream DNSservers
-        direction: in
+upstream_dns_servers:
-        protocol: tcp
+- 185.12.64.1
-        port: '22'
+- 185.12.64.2
-        source_ips: "{{ src_ips }}"
+
-        destination_ips: []
+
-        description: null
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'stage={{ stage }}'
-   -
-     name: "{{ stage }}-awx-access-SMA-mgmt-instance"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: tcp
-        port: '443'
-        source_ips: "{{ src_ips }}"
-        destination_ips: []
-        description: null
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=connect,tenant=management'
-   -
-     name: "{{ stage }}-awx-access-443-SMA-peripheral-instances"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: tcp
-        port: '443'
-        source_ips: "{{ src_ips }}"
-        destination_ips: []
-        description: null
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=gitea'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=keycloak'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=kibana'
-
-hcloud_firewall_objects_backup:
-   -
-     name: "{{ stage }}-database-backup-ssh-access"
-     state: present
-     rules:
-     -
-        direction: in
-        protocol: tcp
-        port: '22'
-        source_ips:
-          - "{{ gitlab_storage_server }}"
-        destination_ips: []
-        description: null
-     apply_to:
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=postgres'
-     -
-       type: label_selector
-       label_selector:
-         selector: 'service=maria'

group_vars/all/versions.yml

@@ -0,0 +1,29 @@
+---
+
+elastic_elasticsearch_version: "7.16.1"
+elastic_elasticsearch_exporter_version: "v1.5.0"
+elastic_filebeat_version: "7.16.3"
+elastic_kibana_version: "7.16.1"
+elastic_logstash_version: "7.16.3"
+elastic_metricbeat_version: "7.16.3"
+
+gitea_version: "1.15"
+
+prom_grafana_version: "9.1.5"
+
+harbor_version: "v2.4.1"
+
+keycloak_version: "14.0.0.2"
+
+pgadmin4_version: "6.14"
+
+prom_alertmanager_version: "v0.24.0"
+prom_blackbox_exporter_version: "v0.22.0"
+prom_prometheus_version: "v2.38.0"
+prom_prom2teams_version: "3.2.3"
+
+traefik_version: "v2.8.5"
+
+connect_version: "10.2"
+iam_version: "10.0"
+webdav_version: "8.4.1"

group_vars/backup/plain.yml

@@ -1,9 +1,11 @@
 ---
 #TODO needs to be removed after story DEV-361 is finished
 hetzner_server_type: "{{ hetzner_server_type_bastelserver | default('cx21') }}"
-hetzner_server_labels: "stage={{ stage }} service=bastelserver"
+hetzner_server_labels: "stage={{ stage }} service=backup"
 
 docker_enabled: false
 traefik_enabled: false
 filebeat_enabled: false
-node_exporter_enabled: false
+
+custom_plattform_users:
+  - backuphamster

group_vars/connect/plain.yml

@@ -33,10 +33,5 @@ connect_iam_user_management_url: "{{ http_s }}://{{ shared_service_keycloak_host
 
 connect_mail_properties_simulation: false
 
-connect_loglevel_message_queue: "DEBUG"
-connect_loglevel_document_index: "DEBUG"
-connect_loglevel_workflow_index: "DEBUG"
-connect_loglevel_workflow_analysis: "DEBUG"
-
 connect_csrf_token_name: "21f4d682-dbad-45e5-b3b5-47d274b9772d"
 connect_csrf_token_value: "4d2ef8cc-f7d9-46d4-b4d6-f20f9dc48040"

group_vars/connect_wordpress/main.yml

@@ -11,4 +11,4 @@ connect_wordpress_oidc_client_id: "{{ cluster_name }}"
 connect_wordpress_oidc_client_secret: "{{ cluster_name }}"
 
 connect_wordpress_buergerportal_username: "buergerportal"
-connect_wordpress_buergerportal_password: "buergerportal"
+connect_wordpress_buergerportal_password: "Buerger?P0rtal."

group_vars/connect_workflow_heatmap/main.yml

@@ -0,0 +1,3 @@
+---
+
+connect_workflow_heatmap_enabled: "true"

group_vars/harbor/plain.yml

@@ -2,3 +2,71 @@
 
 hetzner_server_type: cpx31
 hetzner_server_labels: "stage={{ stage }} service=harbor"
+
+filebeat_inputs:
+- type: log
+  paths:
+    - /var/log/harbor/portal.log
+  fields:
+    harbor: true
+    harbor-component: harbor-portal
+- type: log
+  paths:
+    - /var/log/harbor/exporter.log
+  fields:
+    harbor: true
+    harbor-component: harbor-exporter
+- type: log
+  paths:
+    - /var/log/harbor/redis.log
+  fields:
+    harbor: true
+    harbor-component: redis
+- type: log
+  paths:
+    - /var/log/harbor/registryctl.log
+  fields:
+    harbor: true
+    harbor-component: registryctl
+- type: log
+  paths:
+    - /var/log/harbor/chartmuseum.log
+  fields:
+    harbor: true
+    harbor-component: chartmuseum
+- type: log
+  paths:
+    - /var/log/harbor/trivy-adapter.log
+  fields:
+    harbor: true
+    harbor-component: trivy-adapter
+- type: log
+  paths:
+    - /var/log/harbor/postgresql.log
+  fields:
+    harbor: true
+    harbor-component: harbor-db
+- type: log
+  paths:
+    - /var/log/harbor/jobservice.log
+  fields:
+    harbor: true
+    harbor-component: harbor-jobservice
+- type: log
+  paths:
+    - /var/log/harbor/proxy.log
+  fields:
+    harbor: true
+    harbor-component: nginx
+- type: log
+  paths:
+    - /var/log/harbor/registry.log
+  fields:
+    harbor: true
+    harbor-component: registry
+- type: log
+  paths:
+    - /var/log/harbor/core.log
+  fields:
+    harbor: true
+    harbor-component: harbor-core

group_vars/kube_control_plane/plain.yml

@@ -6,4 +6,3 @@ hetzner_server_labels: "stage={{ stage }} service=kube_control_plane"
 docker_enabled: false
 traefik_enabled: false
 filebeat_enabled: false
-node_exporter_enabled: false

group_vars/kube_node/plain.yml

@@ -6,4 +6,3 @@ hetzner_server_labels: "stage={{ stage }} service=kube_node"
 docker_enabled: false
 traefik_enabled: false
 filebeat_enabled: false
-node_exporter_enabled: false

group_vars/management/plain.yml

@@ -2,13 +2,13 @@
 
 hetzner_server_type: cx21
 
-connect_image_version: "latest"
+connect_client_admin_username: "{{ management_admin_username }}"
-
+connect_client_admin_password: "{{ management_admin_password }}"
-connect_admin_username: "{{ management_admin_username }}"
-connect_admin_password: "{{ management_admin_password }}"
 connect_workflow_env: "stage:{{ stage }};smardigoUserToken:{{ smardigo_auth_token_value }}"
 connect_process_search_module: "external"
 connect_oidc_client_secret: "{{ management_oidc_client_secret }}"
+connect_external_task_script_worker_enabled: "true"
+
 spring_profiles_include: "prod,postgres,elastic,swagger"
 
 tenant_id: "{{ management_oidc_realm }}"

group_vars/maria/plain.yml

@@ -7,6 +7,9 @@ mysql_databases: []
 
 mysql_users: []
 
-docker_enabled: false
 traefik_enabled: false
-filebeat_enabled: false
+
+filebeat_maria_enabled: true
+
+custom_plattform_users:
+  - '{{ backupuser_user_name }}'

group_vars/postgres/plain.yml

@@ -5,6 +5,9 @@ hetzner_server_labels: "stage={{ stage }} service=postgres"
 
 postgres_acls: []
 
-docker_enabled: false
 traefik_enabled: false
-filebeat_enabled: false
+
+filebeat_postgres_enabled: true
+
+custom_plattform_users:
+  - '{{ backupuser_user_name }}'

group_vars/restore/plain.yml

@@ -0,0 +1,15 @@
+---
+
+hetzner_server_type: "{{ hetzner_server_type_restore_database | default('cpx21') }}"
+hetzner_server_labels: "stage={{ stage }} service=restore database_engine={{ database_engine | default('') }} manual=''"
+
+docker_enabled: false
+traefik_enabled: false
+filebeat_enabled: false
+
+custom_plattform_users:
+  - '{{ backupuser_user_name }}'
+
+# postgresql related
+# defining type of server (naster|slave|restore)
+server_type: restore

group_vars/stage_dev/argocd.yml

@@ -1,7 +1,3 @@
 ---
 
-awx_operator_revision: "main"
 awx_smardigo_revision: "main"
-
-jaeger_operator_revision: "main"
-jaeger_smardigo_revision: "main"

group_vars/stage_dev/awx.yml

@@ -1,3 +1,3 @@
 ---
 
-awx_hetzner_ansible_revision: "master"
+awx_hetzner_ansible_revision: "main"

group_vars/stage_dev/plain.yml

@@ -145,28 +145,32 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 shared_service_elastic_stack_logstash_01_hostname: "{{ stage }}-elastic-stack-logstash-01"
-
+shared_service_elastic_stack_kibana_01_hostname: "{{ stage }}-elastic-stack-kibana-01"
-kube_master_01_hostname: "{{ stage }}-kube-master-01.{{ domain }}"
+
-kube_master_02_hostname: "{{ stage }}-kube-master-02.{{ domain }}"
+ # TODO use {{ domain }} agai when moved to smardigo.dev
-kube_master_03_hostname: "{{ stage }}-kube-master-03.{{ domain }}"
+kube_master_01_hostname: "{{ stage }}-kube-master-01.smardigo.digital"
-kube_node_01_hostname: "{{ stage }}-kube-node-01.{{ domain }}"
+kube_master_02_hostname: "{{ stage }}-kube-master-02.smardigo.digital"
-kube_node_02_hostname: "{{ stage }}-kube-node-02.{{ domain }}"
+kube_master_03_hostname: "{{ stage }}-kube-master-03.smardigo.digital"
-kube_node_03_hostname: "{{ stage }}-kube-node-03.{{ domain }}"
+kube_node_01_hostname: "{{ stage }}-kube-node-01.smardigo.digital"
-
+kube_node_02_hostname: "{{ stage }}-kube-node-02.smardigo.digital"
-shared_service_iam_hostname: "{{ stage }}-iam-01.{{ domain }}"
+kube_node_03_hostname: "{{ stage }}-kube-node-03.smardigo.digital"
-shared_service_mail_hostname: "{{ stage }}-mail-01.{{ domain }}"
+
-shared_service_gitea_hostname: "{{ stage }}-gitea-01.{{ domain }}"
+ # TODO use {{ domain }} agai when moved to smardigo.dev
-shared_service_redis_hostname: "{{ stage }}-redis-01.{{ domain }}"
+shared_service_iam_hostname: "{{ stage }}-iam-01.smardigo.digital"
-shared_service_kube_argocd_hostname: "{{ stage }}-kube-argocd.{{ domain }}"
+shared_service_mail_hostname: "{{ stage }}-mail-01.smardigo.digital"
-shared_service_kube_awx_hostname: "{{ stage }}-kube-awx.{{ domain }}"
+shared_service_gitea_hostname: "{{ stage }}-gitea-01.smardigo.digital"
-shared_service_kube_prometheus_hostname: "{{ stage }}-kube-prometheus.{{ domain }}"
+shared_service_redis_hostname: "{{ stage }}-redis-01.smardigo.digital"
-shared_service_kube_jaeger_collector_hostname: "{{ stage }}-kube-jaeger-collector.{{ domain }}"
+shared_service_kube_argocd_hostname: "{{ stage }}-kube-argocd.smardigo.digital"
-shared_service_pdns_hostname: "{{ stage }}-pdns-01.{{ domain }}"
+shared_service_kube_awx_hostname: "{{ stage }}-kube-awx.smardigo.digital"
-shared_service_webdav_hostname: "{{ stage }}-webdav-01.{{ domain }}"
+shared_service_kube_prometheus_hostname: "{{ stage }}-kube-prometheus.smardigo.digital"
-shared_service_keycloak_hostname: "{{ stage }}-keycloak-01.{{ domain }}"
+shared_service_kube_jaeger_collector_hostname: "{{ stage }}-kube-jaeger-collector.smardigo.digital"
-shared_service_harbor_hostname: "{{ stage }}-harbor-01.{{ domain }}"
+shared_service_pdns_hostname: "{{ stage }}-pdns-01.smardigo.digital"
-
+shared_service_webdav_hostname: "{{ stage }}-webdav-01.smardigo.digital"
-management_service_connect_hostname: "{{ stage }}-management-01-connect.{{ domain }}"
+shared_service_keycloak_hostname: "{{ stage }}-keycloak-01.smardigo.digital"
+shared_service_harbor_hostname: "{{ stage }}-harbor-01.smardigo.digital"
+
+ # TODO use {{ domain }} agai when moved to smardigo.dev
+management_service_connect_hostname: "{{ stage }}-management-01-connect.smardigo.digital"
 
 keycloak_server_url: "https://{{ shared_service_keycloak_hostname }}"
 
@@ -288,12 +292,7 @@ harbor_oidc_realm: "harbor"
 harbor_oidc_client_id: "harbor"
 harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}"
 harbor_oidc_admin_username: "harbor-admin"
-harbor_oidc_admin_password: "harbor-admin"
+harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}"
-
-postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'"
-
-connect_image_version: "8.5.47"
-iam_image_version: "latest"
 
 management_oidc_realm: "management"
 management_oidc_client_id: "smardigo"
@@ -308,28 +307,27 @@ iam_jwt_enabled: true
 iam_jwt_secret: "456ae14462d049d3be76439ef379c7c6"
 
 keycloak_admin_username: "keycloak-admin"
-keycloak_admin_password: "keycloak-admin"
+keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb <username> <password>)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "$apr1$nJfFcFaI$ylS3Qa9BWAvhrMo5tWiD9."
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
-grafana_admin_password: "grafana-admin"
+grafana_admin_password: "{{ grafana_admin_password_vault }}"
 grafana_user_smardigo_login: "smardigo"
-grafana_user_smardigo_password: "smardigo"
+grafana_user_smardigo_password: "{{ grafana_user_smardigo_password_vault }}"
 grafana_signing_secret: "{{ grafana_signing_secret_vault }}"
 
 pgadmin4_admin_username: "{{ pgadmin4_admin_email }}"
-pgadmin4_admin_password: "pgadmin-admin"
+pgadmin4_admin_password: "{{ pgadmin4_admin_password_vault }}"
 
 management_admin_username: "management-admin"
-management_admin_password: "management-admin"
+management_admin_password: "{{ management_admin_password_vault }}"
 management_realm_admin_username: "management-realm-admin"
-management_realm_admin_password: "management-realm-admin"
+management_realm_admin_password: "{{ management_realm_admin_password_vault }}"
 
 harbor_admin_username: "{{ harbor_admin_username_vault }}"
 harbor_admin_password: "{{ harbor_admin_password_vault }}"
@@ -347,28 +345,32 @@ mysql_root_username: "{{ mysql_root_username_vault }}"
 mysql_root_password: "{{ mysql_root_password_vault }}"
 
 gitea_admin_username: "gitea-admin"
-gitea_admin_password: "gitea-admin"
+gitea_admin_password: "{{ gitea_admin_password_vault }}"
 gitea_realm_admin_username: "gitea-realm-admin"
 gitea_realm_admin_password: "gitea-realm-admin"
 
 argocd_admin_username: "argocd-admin"
-argocd_admin_password: "argocd-admin"
+argocd_admin_password: "{{ argocd_admin_password_vault }}"
 argo_keycloak_client_secret: "{{ argo_keycloak_client_secret_vault }}"
 argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}"
 
+awx_admin_username: "awx-admin"
+awx_admin_password: "{{ awx_admin_password_vault }}"
+
+prometheus_admin_username: "prometheus-admin"
+prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
+
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
+
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation DEV gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://dev-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
-
-iam_opentracing_jaeger_enabled: true
-iam_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-webdav_opentracing_jaeger_enabled: true
-webdav_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-connect_opentracing_jaeger_enabled: true
-connect_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"

group_vars/stage_dev/prometheus.yml

@@ -0,0 +1,3 @@
+---
+
+prometheus_tsdb_rentention_time: '2w'

group_vars/stage_devscr/argocd.yml

@@ -0,0 +1,179 @@
+---
+k8s_argocd_helm__name: "argo-cd"
+k8s_argocd_helm__release_namespace: "argo-cd"
+
+k8s_argocd_with_keycloak: False
+
+k8s_argocd_helm__domain: &argourl "{{ stage }}-argocd.{{ domain }}"
+
+# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+k8s_argocd_helm__release_values:
+  controller:
+    logLevel: info
+    logFormat: json
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+  repoServer:
+    logLevel: info
+    logFormat: json
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+    env:
+      - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT
+        value: "0"
+      - name: ARGOCD_EXEC_TIMEOUT
+        value: "300s"
+      - name: XDG_CONFIG_HOME
+        value: /.config
+      - name: GNUPGHOME
+        value: /home/argocd/.gnupg
+      - name: HELM_PLUGINS
+        value: /custom-tools/helm-plugins/
+      - name: HELM_SECRETS_HELM_PATH
+        value: /usr/local/bin/helm
+      - name: HELM_SECRETS_SOPS_PATH
+        value: /custom-tools/sops
+      - name: HELM_SECRETS_KUBECTL_PATH
+        value: /custom-tools/kubectl
+      - name: HELM_SECRETS_CURL_PATH
+        value: /custom-tools/curl
+      # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments
+      - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS
+        value: "false"
+      - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH
+        value: "false"
+      - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL
+        value: "false"
+      - name: HELM_SECRETS_KEY_LOCATION_PREFIX
+        value: "/sops-gpg/"
+    volumes:
+      - name: custom-tools
+        emptyDir: {}
+      - name: gnupg-home
+        emptyDir: {}
+      - name: sops-gpg
+        secret:
+          secretName: sops-gpg
+    volumeMounts:
+      - mountPath: /home/argocd/.gnupg
+        name: gnupg-home
+        subPath: .gnupg
+      - mountPath: /usr/local/bin/kustomize
+        name: custom-tools
+        subPath: kustomize
+        # Verify this matches a XDG_CONFIG_HOME=/.config env variable
+      - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops
+        name: custom-tools
+        subPath: ksops
+    initContainers:
+      - name: 1-install-ksops
+        image: viaductoss/ksops:v3.0.1
+        command: ["/bin/sh", "-c"]
+        args:
+          - echo "Installing KSOPS...";
+            mv ksops /custom-tools/;
+            mv $GOPATH/bin/kustomize /custom-tools/;
+            echo "Done.";
+        volumeMounts:
+          - mountPath: /custom-tools
+            name: custom-tools
+      - name: 2-download-tools
+        image: alpine:latest
+        command: ["/bin/sh", "-ec"]
+        env:
+          - name: HELM_SECRETS_VERSION
+            value: "3.12.0"
+          - name: SOPS_VERSION
+            value: "3.7.1"
+          - name: KUBECTL_VERSION
+            value: "1.22.0"
+        args:
+          - |
+            mkdir -p /custom-tools/helm-plugins
+            wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-;
+
+            wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux
+            wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl
+            wget -qO /custom-tools/curl https://github.com/moparisthebest/static-curl/releases/latest/download/curl-amd64 \
+
+            chmod +x /custom-tools/*
+        volumeMounts:
+          - mountPath: /custom-tools
+            name: custom-tools
+      - name: 3-import-gpg-key
+        image: argoproj/argocd:v2.2.5
+        command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"]
+        env:
+          - name: GNUPGHOME
+            value: /gnupg-home/.gnupg
+        volumeMounts:
+          - mountPath: /sops-gpg
+            name: sops-gpg
+          - mountPath: /gnupg-home
+            name: gnupg-home
+  server:
+    logLevel: info
+    logFormat: json
+    config:
+      url: 'https://{{ k8s_argocd_helm__domain }}'
+      helm.valuesFileSchemes: >-
+        secrets+gpg-import, secrets+gpg-import-kubernetes,
+        secrets+age-import, secrets+age-import-kubernetes,
+        secrets,
+        https
+      kustomize.buildOptions: "--enable-alpha-plugins"
+    rbacConfig:
+      policy.default: role:readonly
+      policy.csv: |
+        g, {{ argo_realm_group }}, role:admin
+        g, admin, role:admin
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+    service:
+      sessionAffinity: ClientIP
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ip_whitelist_admins ) | join(',') }}"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+        nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+      hosts:
+        - "{{ k8s_argocd_helm__domain }}"
+      tls:
+        - secretName: "{{ stage }}-argocd-cert"
+          hosts:
+            - "{{ k8s_argocd_helm__domain }}"
+  redis:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+  dex:
+    enabled: false
+  applicationSet:
+    enabled: false
+  configs:
+    secret:
+      argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}'

group_vars/stage_devscr/hcloud_firewall.yml

@@ -0,0 +1,53 @@
+---
+hcloud_firewall_objects:
+   -
+     name: "{{ stage }}-default"
+     state: present
+     rules:
+     -
+        direction: in
+        protocol: icmp
+        port: ''
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: ICMP allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '22'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: SSH allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '80'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: HTTP allowed
+     -
+        direction: in
+        protocol: tcp
+        port: '443'
+        source_ips: '{{ ip_whitelist }}'
+        destination_ips: []
+        description: HTTPS allowed
+     -
+        direction: in
+        protocol: tcp
+        port: 'any'
+        source_ips: '{{ ip_whitelist_admins }}'
+        destination_ips: []
+        description: TCP - allow work from home without VPN
+     -
+        direction: in
+        protocol: udp
+        port: 'any'
+        source_ips: '{{ ip_whitelist_admins }}'
+        destination_ips: []
+        description: UDP - allow work from home without VPN
+     apply_to:
+     -
+       type: label_selector
+       label_selector:
+         selector: 'stage={{ stage }}'

group_vars/stage_devscr/kubespray.yml

@@ -0,0 +1,2 @@
+---
+helm_enabled: true

group_vars/stage_devscr/plain.yml

@@ -0,0 +1,125 @@
+---
+
+stage: "devscr"
+
+default_plattform_users:
+  - 'claus.paetow'
+  - 'friedrich.goerz'
+  - 'sven.ketelsen'
+  - 'michael.haehnel'
+  - 'hoan.to'
+  - '{{ awx_ansible_user_name }}'
+  - '{{ gitlab_ansible_user_name }}'
+  - 'daniel.risse'
+  - 'esther.fuhrmann'
+  - 'philipp.eichhorn'
+
+# TODO read configuration with hetzner rest api
+shared_service_network: "10.1.0.0/16"
+shared_service_kube_cpl_01: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-01' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_cpl_02: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-02' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_cpl_03: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-03' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_01: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-01' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_02: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-02' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_03: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-03' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+
+shared_service_kube_ip: "{{ stage_private_ingress_loadbalancer_ip | default('-') }}"
+
+kube_cpl_01_hostname: "{{ stage }}-kube-cpl-01.{{ domain }}"
+kube_cpl_02_hostname: "{{ stage }}-kube-cpl-02.{{ domain }}"
+kube_cpl_03_hostname: "{{ stage }}-kube-cpl-03.{{ domain }}"
+kube_node_01_hostname: "{{ stage }}-kube-node-01.{{ domain }}"
+kube_node_02_hostname: "{{ stage }}-kube-node-02.{{ domain }}"
+kube_node_03_hostname: "{{ stage }}-kube-node-03.{{ domain }}"
+
+shared_service_kube_argocd_hostname: "{{ stage }}-kube-argocd.{{ domain }}"
+shared_service_kube_prometheus_hostname: "{{ stage }}-kube-prometheus.{{ domain }}"
+shared_service_kube_harbor_hostname: "{{ stage }}-harbor.{{ domain }}"
+
+shared_service_hosts: [
+  {
+    ip: "127.0.1.1",
+    name: "{{ inventory_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_01 }}",
+    name: "{{ kube_cpl_01_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_02 }}",
+    name: "{{ kube_cpl_02_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_03 }}",
+    name: "{{ kube_cpl_03_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_01 }}",
+    name: "{{ kube_node_01_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_02 }}",
+    name: "{{ kube_node_02_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_03 }}",
+    name: "{{ kube_node_03_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_ip }}",
+    name: "{{ shared_service_kube_argocd_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_ip }}",
+    name: "{{ shared_service_kube_prometheus_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_ip }}",
+    name: "{{ shared_service_kube_harbor_hostname }}"
+  },
+]
+
+netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
+netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
+
+# smardigo automation DEV gpg key
+# https://git.dev-at.de/smardigo-hetzner/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git
+gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
+
+kubernetes_with_awx: False
+kubernetes_with_gitea: True
+
+harbor_username: "{{ docker_registry_username_vault }}"
+harbor_token: "{{ docker_registry_token_vault }}"
+shared_service_harbor_hostname: "{{ stage }}-harbor.{{ domain }}"

group_vars/stage_devscr/prometheus.yml

@@ -0,0 +1,85 @@
+---
+
+k8s_prometheus_helm__name: "prometheus"
+k8s_prometheus_helm__release_namespace: "monitoring"
+
+grafana_admin_username: "grafana-admin"
+grafana_admin_password: "grafana-admin"
+
+# https://github.com/grafana/helm-charts
+# https://github.com/prometheus-community/helm-charts
+k8s_prometheus_helm__release_values:
+  prometheus:
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ip_whitelist_admins ) | join(',') }}"
+      hosts:
+        - "{{ stage }}-prometheus.{{ domain }}"
+      tls:
+        - secretName: "{{ stage }}-prometheus-cert"
+          hosts:
+            - "{{ stage }}-prometheus.{{ domain }}"
+    prometheusSpec:
+      # TODO Using PersistentVolumeClaim
+      storageSpec: {}
+      volumeClaimTemplate:
+        spec:
+          storageClassName: hcloud-volumes
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 10Gi
+        selector: {}
+    deploymentStrategy:
+      type: Recreate
+  alertmanager:
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ip_whitelist_admins ) | join(',') }}"
+      hosts:
+        - "{{ stage }}-alertmanager.{{ domain }}"
+      tls:
+        - secretName: "{{ stage }}-alertmanager-cert"
+          hosts:
+            - "{{ stage }}-alertmanager.{{ domain }}"
+    deploymentStrategy:
+      type: Recreate
+  grafana:
+    adminUser: "{{ grafana_admin_username }}"
+    adminPassword: "{{ grafana_admin_password }}"
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ip_whitelist_admins ) | join(',') }}"
+      hosts:
+        - "{{ stage }}-grafana.{{ domain }}"
+      tls:
+        - secretName: "{{ stage }}-grafana-cert"
+          hosts:
+            - "{{ stage }}-grafana.{{ domain }}"
+    persistence:
+      enabled: true
+      size: 10Gi
+    deploymentStrategy:
+      type: Recreate
+  kubeControllerManager:
+    service:
+      port: 10257
+      targetPort: 10257
+    serviceMonitor:
+      https: true
+      insecureSkipVerify: true

group_vars/stage_devscr/vault.yml

@@ -0,0 +1,469 @@
+$ANSIBLE_VAULT;1.1;AES256
+30643065666638323233653136336633363138356431356166313531346166336565626666333266
+3430386162333535653837313831636234666138373938300a636264613166386464616231343366
+61303664343865343864313937663930363137343164316461656264323739636434623262363233
+3462616634386163300a313164303939376431396261623039616334623835336538643363356234
+38623336323730656164363263323466373035326161653636323065303933373837323566313134
+33356663306435336136666665333035666133663035346466333031633530373339633034636665
+30633561626637393666653133363636643261343163633062396133626335343837333966366334
+31326664333837323437666464343966396461393461336538303032646531393065656435616631
+34326634616334383166373437626438626434313664326430343966646465343235633835353439
+63616339333332653833636666313564386339653139393536376339633561666435666566346434
+38616636326462393634636336646362613331633331383936613966393334623037333362636635
+32613438663834383735316665366461396130303831643465333364383630363261323034323665
+37653433343961666666333034626534313338323035613365636131396236656163646362303537
+65353036386538626630343765393461336636623730376466663437663638646262346238613066
+31623463303166663838356363633835643233343930383031646438386661663631623233636361
+64336433363563646135333139656463633039373965663333363137616133343862363537306465
+30316437313530393132646234336230663063313632616236636366643332653035643462613632
+33343865373132366430306364656131366261633537663931316163386434316335323864323864
+65313565316631333166396530333161336236636461303964636230646338386533626334356130
+33326262333866653233616265653466346566656436653633343239303131313833353430333732
+31363433313636333137316563373833656561323538656636623064333664633637386636373139
+66633462393137653633336233393536306232386539656637336363303434646633373433326265
+66353765633966326430623763376335613761663130343165373131366664653839656239653066
+64353163313937616138663566663532356661636564323630633864666361383032313634383734
+33636464623532313935656430306464663334323236646531306562303061353836353863636163
+39303531613239616365366332343364383738353335396437366331643666343064633935666234
+62366135323833363431373331363732376339336265636336346637356564306435306530303534
+36323534653136316539616238336437393061643363383066356361383131656266613134633935
+33383037333236623635626666623234376561326536336530623838306435343135313761346438
+37633763373862343930343131393664616261353130623062636465346164393637366436346462
+32643430343064326335383430353563316531353461376666353230626437373438343932636630
+32636234653435363662333932383930303437333237633164313735356463383962376137343662
+64653861323361623435383366616530636130643661336663393333303165346435636335353934
+64303839336563666331326630303236373139326565653139646564363864373739633632376165
+62666263323362653661633437646662336464383761653332616232366331633731313366376532
+61633439313338303565323636303935663064623637666536323561396365383065643636653763
+33313536383039323735666462623135643835333735616430316430326631373164663965393566
+31323235373739663963373465343534383261633036653663386332616365343363663739376636
+65383766626362663133626638393536646635646130323030633830373737656164356338366136
+65343761626136353631323162383438656163333936613732643366396566393730306332613463
+32386137633962616337663065393965373761313035333135633164636332393133383137653036
+62663434636163376130386162396130653735376661323064663264643034393466346339633230
+37626232306664306436646337356365653936636538663965363237646636663561626561623230
+32383165323934666462363764316630346339346261353865313866626135633331323833383439
+35623936346164656565373338386463656262336632643636303864663964383739313164613663
+30383833386662333266613137356233643335393335666432386466306330313266363362626430
+33663162343734636335363639363039633161323631333161373033353732316265643634303166
+32306630336239376363396566333265356266643861353435343064356532646164633165346135
+62396161613865633839343630343436366166623537383239393562653233323239356534346364
+39383639386263613564303834613138326339343838336231303531313037663131363538343666
+65626365323434663636653138353862303764306338393034353763626338323032653336663033
+30656665386332303736303765656130393133636336383131616633643536396633313364663361
+64333533653838623338326535653261346336396331623835363165363231653630613061366663
+32323463393532366532333630376233313437316239333130353465623066643932306466323637
+66326362666131633533303836633561393862356639306562386466363336333030363037306665
+35316333386538636135363364626463656234383836613161633537383662643064336133303662
+36323034663131626262623834646463633266386537663261646339343665623336663337313465
+66616264393263663365663937393266633036656536636562663437626630376365313561306662
+34336365653466313132613462326361663263346135336261333766306339616261653232363531
+37646165363338363632613331323664306130643832636336376438346664666138366432623836
+35393531396261346138656562663763393535646534633565333537333361636264326231386665
+65383864663632656636393038333766303935633932333266376239323565353164346138633738
+35303237656265333133316464356564336361393536353262346462346135613833656532396461
+32393461656163346565323732653761343332303533363530333535366236616533636366613038
+61343065663234663765336437363864313064646338333864333637343639346238383766376464
+39616562663764306639323339356336666363373635636565613038663936393464623938613635
+33623139313838663764373163376238393334636236383866323033633963626534656363386439
+38353435663037353338373533373539663834316439373136356136386231626639313732303661
+64643039313535613066623036376665306164333962336337633031396162626664396262323463
+64313635633430643735633039353663663662353434376331313066363066656130343331336363
+61373863353535393632613038346339346533653031313636336263376564346138336466356234
+35326439383061366433366239623030333236363437646666653038393039363730616132646532
+31373764326131396630336135633831353633626462386266613035666337303932313361663364
+62623438396166386339663563623665336537393066623863656538313334313466366531643832
+30633331393838313137636636633235393563383366323365643737613237303639336434356232
+33373464326435396463373165333039396239333835356234396136346663313062396537653765
+62346636633133383635363236633665643038333765383733343133366363363332393934373362
+30646265346537646435363831636265343834633739333730386165306437636462356534363332
+31343636396366373163326334386135353430653264303132343439323862386238363132633932
+37393461656561316632343837346564386537373262343439376537666639303635656165306236
+31326131336161373734356233623231366662373266356531383861383361313537336364613934
+63333030373335366435666437383933616236303263333466616635396138633466636564313335
+38626534643236623838636637633330616161663663303365386332353030313162643332393932
+38343038666332323430396437663563393963356432336363666666393861363233626166623636
+34326336323035613137313834303462313939653161633531323666353335636534393739626236
+33363966346636376432643530363734326530316638623962333335373039326235383539313530
+30616636313464313862326636386338303564653834653963613233313165626339343962353164
+34616134343363646465623131636132386564363731623631396333626539623962356133343164
+31366236656334313930333834333039363833396135323932333130303730366638613136353435
+39333035343936346662356233346238386433363736643339336330653733356537333662613030
+34376634393738663036666565616330386535313833373336366632323032626131613834643265
+32326466383362623064396662343836303838623934313665646666383835366132353261383464
+33326130373963366331333939363664663031336531656663653265333564303035363364313162
+37373136376339323766396165356261323864616563386563373431373238346264616465333266
+65346661666530356135396566303639303861633233363435343262336663313337313437326133
+31653266363761306666343737343861643261363566363837626530636338313666323135653264
+66333332653734613637383964383133383435616638633639313362393839333832356139396266
+31343035636464316332356161616364373532343163393765376466636561623739623038333935
+34313738333962336638633835316565666162383265346166383138636131303931306530386130
+65396533366233373361306234356533306330353836363764333530643165316266386530326334
+34376431656332363531306537323361343065663132393839386366363837616137653035306434
+33346136666365613530366461623663643239643235643966306632646639346536373563376538
+32653039326164383633653165316364383961653730396539646566653630633139666333373038
+37643934666230366136303738366366343933323935636335636235333866633365326433633164
+63386163316361313836653637353836626238633330633731313434303462613962323362333562
+32313439626466313231346435626535623363363966313631393538353166353431323930616462
+39623337336431666264393861316261636635633961656636663462643635303233323935396138
+64316334386535353331376431356438323064373538386630656238623734373765316266323264
+33323964666564333561643936353262626333303834323761383262393865393830616138326263
+30616238393864656463383233316138396432303538653061633433393535383565626233343961
+31313835343937386636383865373134323433353433656565313932313564333339656235356366
+30326533313838643033383763373933626339323533303037663262366565323365616663333936
+31316334303832376333303962663738363437613261616532393332616333396562303861663363
+64393034373865383866306130303533366132383562343664306238303861326166633830643662
+37653065653766323236623539666564613931326633386231346537353232323635636432346364
+66316339633164353432356632393863323537323838646537666433623864373636643931663834
+37613265363333613863663161326637333637316331336133313333343963383834653038623866
+66343735326463326538303639653764663439666334646362666431313838303139356562633235
+39653135323835646533303266306263393837356235323038373739363061353931396433326530
+62373639666662306430343939376631623331303765386631616438356532636566613866346133
+39323166383230663531393632346632623563303662303964366432636233313335313763316337
+63613734396330303236393131356461303837376639346436313365616237366237363461313066
+34623834333138353232633235373231643633613539663265303733363234323765633265366334
+37313364336238623065383936653461633761363938636632623035653636666335653061643261
+30366137366434366366616331653432636466636363333539626333656434633937633035356439
+39623130393936356465636264623565623534666461353036356436313736333131616361303330
+62343435383132613233666662653635383865306166366235633961656139623530623362353163
+61643939636161346436353334663537373838323737313562313631393639636362363732393263
+63383366366661313735623839363664383638663065363337666562373338363539363662393566
+65313865393730303830656363313761346531663733623131636634346432636162623431323237
+61386333666638383735666264366365383065666334313839616364663531323066663932306165
+62626237393361316336616337383765363566383866346635303436656136663762646336626165
+65393565356635323033333933633236336366646331636530383463396661653361393364356664
+32616164636436323939306232363533666666643632333636326336316130646161643837383634
+61333133313335353262323935353762363439353836663063323139363030653632393236623931
+33626235326561343265393832643530643166376334666665623633363066346238313331623633
+64303538393131643330316361633337376331613361656139383663343962363162326566666365
+37316435313962346664313762366261393037346666363836376233666231666162336264396365
+63646539646562313537333737336366633435343231306539656463613132643063353962326263
+33336330386138386631363334336162326366373238383465663533326165366538623330376437
+35666633626234633062383339373966386535356532333733313633373964336330643964626234
+34653036383836303832313365386363373834326664323539383064356666313430316437356565
+33393638356636323338613161396361343831333534363963383137393837616161363265636338
+34613731653435376561316435613462386436613333383966633034356565626365633235396265
+62313131363665636365653234626437336163643439313639386463303961636436323932316135
+35383739663564383037633735646536393234666439383733386464306561646437626535366565
+65353532663137393433643436346132393562663135393266373631346536626163653465323938
+33633766386435393864623337636136343133383431636462343564336531333031353339623033
+38353738306366643964623639626135336661306264333836623564306532626631333635333237
+36356638363133663564653837633366306134653330633337616330383063343636633233383961
+34613238336332333362333363363335353566313161356133373436346238633266363966393466
+35663961346563393239366565623539356263306539656638353830326666626266323663383261
+33383765386265363533656666316439653530333933343034663863653861366262316366363331
+33616239663533653334646134656135663063363039653961653064333432333738666132643334
+61646639613733666361343831623334336663643864656362666166346437373162643735636234
+31356463376330633461336366366263366466663935313338373834626630376363313831373036
+30643732323930316261373539623866303562336239383536306161363361663337656163363038
+65343932393139653433306537623933613439666362383337663135303232393535653639646533
+62393438613939313762613164323264623032656230313966373432323931356232343932326438
+36623561326632313038306333663230393164343264343830353962323933313537303634656362
+30626437366430376332346139303435386363373930363962366631343464333066646564383061
+31343834356436363130333566373936303764653763613063363536666664343637333337383037
+36306333376539373937303664633139396533393866633235393439326332646132343764353565
+64353262346133623034343066316361373561623634616262366138636565356637373634383135
+30366463626130643736653633376562376433376532393038633933643836373631366436356561
+31333433313039666336663337353436303463323137313666656137613538663231643565353035
+63386336346566383764313038303235333962326235636331313637326637353362636638343835
+33393135626539383538396361666266303061313932356364356436666564626238383639323637
+30643265396364393336306338366364643337303365313335323835643631363762393733353664
+64316235313864653434626363636537373562333731353533323864363062633663656638643433
+37623565353764646338666435366264383538636338383937376437643835333738373030636538
+36363265363130323930353235393731393432396661336565653235373130643734393138616665
+32343337343932623464323361313436373161336663613065623063346533346466643066386330
+63643366343136653264383432313263393864383438623863386230633434383962323762663465
+34623033326438636131633237616131396437623264653831333634323134653338613765663936
+62333435343461633939613365626637386534633335666330623536373135333761646339353934
+61373762643439616331613634613963316265323331633130666263623462613061646263393763
+31376166316639666334343466633738353931316163366635363737313738343439636532393361
+35363831643166383765633462363561376330396663383534373136306430376661376438376565
+66633562366164643535633633353838663162636537356131353265356436393531663862653734
+62646535663761323439666535653462386638303630333862303965323761373237373266666237
+37353636653734363634616461333062373463373861386436366130653532303934393130663863
+33616138383234623866333432316164653564656237636138616365663735306631653434616535
+64653561643262626162343363353762653632363338363832323436303636386265653138383230
+63366530633934623839666434613930626130313537633734383865636461313034613764343039
+37343566623437656539333534383136333737363035303866323562613937303863356635663636
+36383265653532393831666337303437316562636132663230613239303630613034666635623535
+38393164393965373233623266353133376135333038613462653566666430633963323162396562
+30666164393736343033333736326165643964326661373037303364353531376132346461656363
+37313463643936353830346434353132623662643439643539636638643731383133393830323635
+34383464393535396263633936633665316463333466376465633534666430336132646361386138
+38653330646435613564366662666163643464373539393830356265643532616365333431303532
+62363432343739366433646136343731363161646331623530316333303762623438636531613561
+37663264656539323333366230656536623663306635353566336338363638353732303434333134
+32373065666337393632303063613135343030333935343337663838323361386564656534636438
+31666333363461633866663863346162636131313566323830393239333563306662626337636661
+65356534313462386366643261343066376530333331326237306530303938663561343739656332
+66396663646438323666343230346261306438313430636135383163313336396639653530373665
+36393637616534656566316230663233626563363965383861323961386365363838626235353564
+33386165646663323831643964653666636662636133313666353262366232323131616462353966
+63326531376461333066666262336663316361653733346166393163623039636330643563653165
+64616137396435646135383130656132383062623930346337646433313838333433646133313935
+63383536326538666533393332393462386433653231326435346432336562353066313463393966
+38366338306230373261303131643532336563303533316334333030613830663739366463373934
+65363039663931616637623664373665353062373330326634386365316462623166356134643634
+30353265663834396366623634373763313939353833373133383833306131383737643566633930
+37316235653237616566376232393465643737376164303138323766663537373932633431313465
+39316232663932386262363262306435333233323233383032626232636130366362373332613663
+31323463303264363234383931363435663834383562636166353936303433386439313435626563
+34356161643535633335376336656338633632356266383330306562666462306434623463363836
+36636134613664326439316437313135643664353434343264666562373463616630373135613062
+32366431653134323738636433613765646530636432313865626132303163373835303730666233
+64646135376130333639323533643633376632306562653264323635386233353038666439383065
+34363735356234366132643233303534376535393739663437613032356164313835613133623733
+66373830663233643365633536626161346136336564316333313563303235336262323831646662
+39316236653335643633613334386530356161613631323332306135653731336361386135636566
+64653131663036663565313031323135663031653734323931326435663432356534363762376663
+63626261386361323835396637616235623034393533336338653430373237393466393037363134
+36393930303664306164316364643931613736646433383635643564333634303939376633663362
+35663865623135623735653761333164663665343163656438353238393561333930343262663263
+65353261633037363563646662616466386333323430623562383732666166393834626339343766
+65373935343436643366393264323337363438393532613965303261373332366663343163666333
+31613963393830306462363863323034383236333630323038343934313836646564343734323835
+31646439386132626465316534633432663665643561306531636236366432353131343630366538
+66626461393338373232633930666635636537376465663963366131333364303133396138333437
+38393130333932333065366136616633326338613737373233303862316563623333323037613861
+39613938376231653131396331396261353666633161363062646261336633366636303536383635
+37616436323932333464313834346634396662653466323132383764346437353865363064623162
+37353963616430613162323762303866336436623531333132643933316335316439626230343539
+66306665643739373764346538643663353439633638396230366432363837356161623339643863
+61353337336239373337653462313435383033623761333434343837393339616431663763623632
+32313832303539653736653764623361383961643533313733393061336663333137313335643462
+38323937336365653338656137383937316366313562386366306232646666316663616231653232
+62366564656334653937633565363932383137336137616163616365656462623338613363396137
+30633165326533353662633963643266376532363332376232613266326334396433356635373264
+31303130643635343937653831373233376639373137356534353930366364366239653965336261
+30303630316561616564313231356234306535626264623162343131666162663864336661663265
+63366332363761623864343032656261643433626561663738663036333336313638393263653633
+38303566346632313435366562336330313437393663393432383235313638613130616161303038
+33623064613266633565353739636232363865343964306563626463326265656138356431663731
+32383362353934333931393264363937306263653164373332386437346237636638616331306530
+66616239633733633439313735343963633632663861623734626132663638333633633739323365
+36353966303930343132613663613138353538663537623833383939633264373766303063646464
+37613162653362336539323938383061613066663565343839393731303565353534633766643563
+37323134623738366131343139386330383532633464333634326334303033643839656466356134
+38333364393538323631396461323839396364346564653434316531393633316132346165613565
+66653865343138376430313466393135363030393662363834353632343966336161333364326533
+30363531383338333331383962343237633463623332366561343336363336373931353162626363
+37303061396239353163383766386332646464666637343362636536333333653939323435653461
+63666162333461633236313863313561363031616364356130636438636632326531353732396331
+38656566356331316162303065663035316634656563393132646134336634313531373136633366
+63333236313232646561346238313064623738313263303164663036396536643066333861636337
+62353033636134636131353334373633613131363336326531633433326238323635623338366635
+39313866333963346334623038343232323765623661633638646537393334396338376438323964
+38336236383638353336306362356363373763653562623639663037326237653238313634373830
+32666130373631393564386632396634303236336339346236366132633333333735373233363433
+35663264646462396634326535313937333838323665343564626336656535653063363533343433
+35636131636161643662656261663132386632643061623065366161386466383931653236316233
+30646333396639363436616162366436326634396636303964623637306531333037346531353337
+34353064373936323164313465303032313563636638326237666133353433323938376338643563
+32353336393431336232363064353638663930626638383438616565316161306530663165346431
+65636363396366633735346661376339333334393363366239646135383332633366333831646239
+37376338386538623865343061636163643936333134323930386533383335303638653761303430
+34313465393837366264633539303839633138383230613634306131633962316366343937356439
+36323566666162373763373564373938383539336636313636396630333033666564636466353931
+64643330636330343232646361623030636366323236373236663966346333633439396439366366
+63653537333261373938643637653539633163666137303130306166313734333433396361363932
+34376162613935653563623736353038616530366537623238636466343137346465333162346334
+33306539316635336433373334336462353165393839643664376633613162633462383537336463
+37653938373836333964396131356638326362316366303763336465653932633930336332613237
+64316134663764366232616438353038386562363962333731643065353839666536663131653632
+36383331366330653239633061323861356565656438383262333164336333356630663465656337
+65663037396530653566303936613364666265323935323664633962666430386361353632306565
+65396562396334623366636663636163323435396534653962346164613365653935383635323833
+62336264333563623133383061353565623438313336366565633330386161383061386432663365
+37373735346639323964646437613732646236396561633065336562353263633630343264646162
+33363964343336326636623265623237643839613530323366646266336433393134306230336536
+66616262386232336331373239333663643864346461343664663536643836363564663065663434
+63363733663635613530643832646661366530616536633333316563316630633732643438386264
+65616139663731633661316138633266333936646138313238336164663730343131316336366465
+39663131313131623964376265653832653134366532643631333234613631643762356136343161
+63663837363566363964376234353338353661663266383863386431636438663231303338626565
+66316439633430363731386663303334663330643234353065613132316538353963323466353438
+66326165356538653034336434636539636162333032306139323032323666313032313436633131
+37366632373236393231313938613530393763336434363733386236636331633531623266616664
+61333962343736393661656533356663653438316636643266303363393463323934343939333734
+35323664363363383830373537303137393932326266333634633937313233303536353537336432
+30663866633934353335313465303035333363626233663661663835313262336664643166326165
+63326237643935353539336433663363623831646232363338323935623535396531636630346363
+37343939646636323365336130666136646334386232316166313162323233646433316432366233
+34643565373638393036666466633039366232396431373637343662663831623231623062316432
+64373938623135613565393533366530363834633831313031346564396366336465663639303733
+34396338396665356631303938343632653638376233396665633434643530623030353464386531
+30353436383064663832346230333231613037323933626535633765373630323533656662643836
+65363965363561616336376339363865363562386664346564373231313737613937373035386633
+39643664396439323439373630643931663337393066323131356436393939333239666338303666
+38623939636630633161333038373931343030663964633966386133643235353938666139646239
+62623235333561303839316165306235383836336331376638326339386361656336646234373932
+63646361323665346530373135316230653461646333356461303133333561643432376532643765
+31656366396638366566366664383231323339366165383135363734373939303362333261336132
+63373364336265643739303364326339636162343730366433616233373665613963383061633862
+31393132333734393866326334656533663133643361386131373534343361396261373032376533
+62393836373164336630613436323230313938393033646465633239396534613561376365376665
+32313966396131623334376232313436353833326663313663303336313062633034653861663138
+36643464303735376535393833373439656237346537663438626166343065396363326463306665
+61616166376162333265373332663630386435313835336330303361666465633739646338646535
+34623435373763663434306262316334323331656334323831303564366431396238356232623365
+63323338636262303134396539656263303239643531343065393035373738383630646437653231
+65333733643432616633353238373630323161303664396531656663373031646231613735346538
+64633336363434316163623234646439396464306533343332386333316265353534316565333435
+33393266666561656432623738343465336337376366653465386334636636666466323162353830
+31313434353439323834646361383035613338376136633733623539663963636630643638386162
+33643665336465643632363134386430346465336361356436353831343134376131643133633065
+61303536363762333332616630333462656462353235626463643162616135356432343538363735
+30386634613732626331343330346265623636643734636638303937626565373332373762386534
+37386337376165656433633537343235636232383530316134633633363464373064656336633464
+33303662636565396532353830353565306566303231616662653533396364643832323335356234
+63346232626437643838373834343562376631646430643632363261613334646530383539623265
+63323534623262636437313530376165383036363134353432633935633139323832666566393063
+62656466633335363334646262323032646334393065363465353866303137656430616135396530
+31346166646165656665393335626664303136643735646664653037333263383437326463303333
+31393134323132346233666563333036353832376534326338633261303636666161393536303039
+61626633363065333433323331633431373764616332313565333634306532666461616533393436
+38663939326532333638663832373064633665313637663864636432636530616666313631653839
+37373935646231323332653339363038616133383463393530616239636665646562643431386339
+39663433366261653662313065376633633765323531336135386139613630343565396632316336
+31626465343564666537343761313064303235636563366133613434643961363437343433333939
+37653762623565666633333637393630396564623535373562366132633931653261626135663030
+62633937313738663334656366363137366232323037343639393166366665626366613633303565
+63303061343638636631373338663436656439353533306232386431623062373763356338393935
+30386432313436336336656465643134373063313634366336323564306237333039643266613230
+64643863383036393065313236303236336664353361306138323439616363323735316136623438
+65396433333436383263613363373431613663363032653939636436373437346665393664323433
+32663935653866346535316337376665343165373731373764643333333466383165313735646437
+36376239313162656134366139363432316162373035363432353963323437333764396364626636
+61646663346333353865663530626531386263396137303131626161376137643037373235653631
+37666139653736643635313861323433326136313762666338333235383938363032626164343835
+39383632323233333830623866616463346138313063633436353231613937643138343634653333
+37343939646333656231363236353832356165343364313030623832313965353631613039346461
+35363463626631316564393332643961373032316662316330363362383866303364373565653333
+30636230376461313136666462633235383031623232623966313839396639653930353661623035
+37323263666535306265353138393639366330366137323662323361353432636262383765653062
+35303565336236646538666634343262663634366332636663653632396261373236386134393561
+31623231663532306231656362333765653739613438333164343733656163333836656139373266
+65653666393366643164666539386338346233623438303931376333313065313135663262303262
+34333961666463363565346462613836383837613865336430336466666437646665323365386233
+31366661646531303836656233353565363138376130336265643733386331343264336534633235
+35353837343666623664343039346533353964313464646666626330353839303731366639313465
+64303866336461626161353531323163383962306162333061376439333064613937333331393863
+63313331383266323461376530663335313464636465653065613535643035326536323339393564
+31313032616435636364326336303661303565646537353337373166313265343331626364343861
+30343066396261316661333435613034336362343637353137306336636361303166386164393137
+39363038646332613235623664333562653234636533393334326463346430316463613134653739
+62313266653034643463633037316666386335393937396132616564356331346535303439373638
+33396234363433303638373030326663643031333032623136663330386532303835343361623863
+64346137616530633438376336306362356136376438396434643639376366366366643730306431
+64343931396333303261303566313831643936663263613036333262346631656234623533396134
+31373330333430323866306661663635343664623762386563343233366535653937326562633265
+38396561376263613039646536633866353830383837356464353937353030373030383430336131
+31366235336133333162373038393938663139303266313166663064613830313364623663343431
+37396137396139386566656635383134343231366161373538653466616663303738626434643738
+65626465666236616239316130346465633063323437303266626530313561616433366134366438
+32333437363631363731666666313466646635343765633133393630653764613038363233663332
+61633666383966306632363937383935653232376362373966376538363861316135333963356239
+30313262323831623031323631386466353831383033383961663661383434386237376331366664
+31636635393930303130663663346439303531653836643031356631623664613665386438306337
+31343963383830636637356165626662303739373439643237636164363634393863366437303135
+39373932663365663864626132396437616538333331363561396263383035663362373931666465
+34613464373265653963633463373635333333383762663164643636633838666161613863663131
+65326537613366333062363835336262376563323735323633636232356534333766346161663933
+65383763303663653034623837393066643935383036383163306331346162633234396561306465
+32396533613566383461393939646537333635376363633534323335326165346136636138336231
+35623337613335393238323138313631666439323763366337386230343732643337633934376631
+34633638336661633132613363386531393064353032626363323339663561623138373832363235
+35336530663934666433633635323462343130623033653837346165376631613837323137313033
+38313738396462616533393365353033373366666366643137306131316538653761313062383234
+38303863356539316639666163343332316666633361383133616239633063393835396339333932
+38386439323333326637383838633266353537663364363139386437376538326265313637376465
+62373430313932363138626537393938316633616635393062666537666239336365323430666637
+34663634623964653838316361386563336464616564616236636264383438613434386631333632
+36623035646130636161353636323839626666313431666464393336663838643032663735316238
+64663534656262343132656337653135653632653961633333373864326235363635613639323131
+64303036336161646431383135316564316230386264383862653264383037323930616637353335
+34363239616261383437626562306239636332393931326535666566353763393162353166363031
+61646461666639393561323561613132643163326330653230383861336164613433373763373464
+31616631663462353039616432626662613161306334646265373735643934323837336464626232
+64343766656639313066656239373631303331346330383764366531623934323233366432356333
+37643039306337386166303038353636393634363331373936306636663364366664363339376330
+32616361316630666338383463393239333137326165346230303566666333666232356530393238
+30313937326131633765613732326339316631653335633631313931333434323932323334393431
+35626437643439613838633063616562313362303133383164313933643533306163333364626338
+66376434333531393561663935643033656265376234616466656230376636393330346332383031
+62303732333535643233666131313362663564613434653366656335373434343938643638653934
+32376338663837363538653030633133643561326165333363333563386561623162363231386234
+35356237343166323733386437366661326464663061303061366430383866636236323738666564
+61306135383631306534616537643563663437653166353533656665346433636432656635663566
+33613534396561336339333335303861386533373436316632636661376438353731353030646161
+37666361396161633766633765633066303662663233616163643963356138613635636638313035
+61373062666361353439396438393234343731386434633538633631663661346161366465663962
+33626438313562636138656238303934373038636335363764313030316134666161396334396236
+61333262643764303066343065326135366530393337643434373335316639346337356166393633
+63653133396365366135623135623232636234343330353865616138653261333133303437633733
+61336130313039376562356561356132396164383365383632393335623132376630303161376566
+32626566613364646466383563333135343064626537333466356134303635313030353537663737
+33623464373536376330366164383139323664613739376238323334316138333264363637643135
+36316332393439313438333064643933396335386137636637393435623937373634616362626464
+61666539653237373864323435313362353466666237613631363538393362646534646166326261
+39636130303335313233313762393366636163646535653362663863663630366136616239653861
+33643630613364306562363962643039316237313964326130323736643331333561356665666131
+30626365366339663065363461633538323235616238633633313264306332316534303466623339
+61656431313035613665306435363933306634376263626334616137353436626537353662303364
+31306262333463393264626337326131616337653862376335306263356338333966626537353439
+36343635313965323164303066313865346264633637613161613661323963323236306133646635
+34333034653630663161333237393761623136623631396234373533633037626433666533303162
+31326630363431663366306330316330383863326236383632363363343761346165333139386335
+63376434326331653133623761323431326363326431313464653664643431303361313065323762
+30323063333931363836373262343431613733356631386161663466373561656462636331666661
+30646563303034353038346632353835663064653939353035323232383635343232666133623430
+39356339373235376261323964353532363139366563386230313337393962626434376539346431
+62643964373161336662336131616666626134643038643464646462323764393438626635356361
+33636531343239373163333763316535343832646231666136373163633737356633346635623437
+66656633656461356233393366353565363430666237313939663139666466333539643961346336
+35343365646132346337316461653631343839336537346238653032636362613339366634356336
+34373937303136323331303833303862303931336261303035613931343333303932353031616366
+39323439383663643930653635336133373566343334346435363037386534616231393737626333
+30323331646263663039393262633236393933656135626666366433396566646565303337666461
+64333838646164653934653431313233343133643865353165636339373961326565356661616639
+36393338666632316666303464343235343036393537643131353563373765643531303230353162
+38653137646561376537626436316133636564393562376164313337633436396439613233393539
+62353936303038373364306434393265623236623264366165646539373231366333626566353466
+39626666396634306631376262366465326333396664613837356232316663623566623266366336
+63643630336665663262336364646363313964653531323366306362393061623166616633346632
+66376331343364343139386561376663356638373832326435646130653665303734646163663538
+34666433643763333835303966353135363263653730373464346238666166383864336566303535
+30393861393637383433373832666161393938343164353838383832383433373264366434393662
+34393065333461613934336535323961663662623536663832313265653039356638303336633936
+34663365666231363138613366623863396330656238643764653366346131613162306433616464
+34376630373832616337643565333036653565353030613563636537393339393762653462613765
+39366135366162303164306261376437613434393535373130363335386563306363343465373534
+31666265613636333064303034656639373830333437373664353536343838323066643237326330
+36306364653762666335373630303262643739383037386565613739373664663335393033656662
+36356437323165366232646463663963323164616335626438303463356263666266636561376365
+62353161666533666164393933636562616366336138616366633632333736666336663337356564
+35346639353466633635356362316135663532393434636565306363653562383732616533656566
+33663162663839333462383863643863373638663938306265323830613331643465316236613732
+36393936326136393864353564663433326635386537353439333064333932626132336339613830
+31393635613832653630643865663531356536393564326465326630336563363430613161313436
+65326264343435396464353665303339666666333431323663616339646261326663376663343361
+65383634323332386332303536366134646462636463353338613363653864346636383764646533
+34333533303137636530663666366232626261323539363239613632363930653730356664643635
+37643737326230633639333330636135613938633464393964333431306432623362616530353364
+63663666626530303030653365383530343133366136353065346336356136363737316132646139
+65343336616562626364363637393836633461373266323031616334346232613231313337656163
+32663666623062353266326566666238626535663739336234316535346361346438303139613839
+33663035646563316564653137663833643730313934666530656431313833386465363566306334
+37316664666266383666353966353062376434386566653032343635396636356631613961373938
+62303736343836396433326236643963656138333265373137323734303963613636613534363434
+30653334633562666233306131313463383162303631353362633039363931313066643261656334
+62326262616438636266303731386461643966383733396139343233333138353936333933643139
+35313166323037623366396364343838323530656161373031363730613831383164636239366661
+65343131323532313133386236633638623933666133326338393463633635386235613436633730
+39616564666530373538633861376533333238366637663531383665643565656631326438656362
+37303866343637316462313939666131653635666539653864633566663662656662616338643863
+39666336313230346230396665396336623334643538323634306637646337623666633165656231
+65373930633133623466376330316332363236323733393461303831643737626362313735383030
+62643735316162326439623265326562666562323634316562393438303830616266613566336436
+30613931333339303964653264643332323836333836363566303635353565316331366230343133
+64313039376334383766

group_vars/stage_ext/plain.yml

@@ -1,6 +1,8 @@
 ---
 
 stage: "ext"
+tenant: 'bdev'
+hetzner_networks: []
 
 docker_enabled: true
 docker_config_enabled: false
@@ -8,7 +10,12 @@ traefik_enabled: true
 filebeat_enabled: false
 node_exporter_enabled: false
 
-# TODO read configuration with hetzner rest api
-shared_service_network: "10.2.0.0/16"
-
 shared_service_hosts: []
+
+# Note: all dollar signs in the hash need to be doubled for escaping.
+# To create user:password pair, it's possible to use this command:
+# echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
+traefik_admin_username: "{{ traefik_admin_username_vault }}"
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
+
+shared_service_harbor_hostname: "prodnso-harbor-01.smardigo.digital"

group_vars/stage_ext/vault.yml

@@ -1,18 +1,24 @@
 $ANSIBLE_VAULT;1.1;AES256
-34376237343736386538353235346231326462313534643130616532633535613331643236353764
+30316130326434323533613836303239636361376431353133363233333566313135346232663534
-3737383533313861373030313237366131356438393333350a323230316663346634636634353239
+6335633261323064386630363336316635636537333238650a323738333831383963363031313338
-61326262653334646539626464646663383164666166306162646166333462383833333832353461
+34643139323365643561313637623463653238316138656437346632656532356330323335366464
-3437663431653566650a383632653134343238393762333131613633313036636536343831333630
+6436363531346137390a343633373630626439376163623331613139386363303461323136636336
-34633361373264376263303364353531636434356263663965626639616666633861636463383637
+30343636646366303737663364363364636266643731666634643134306430356338653239663037
-34333838663834666532366564396566313739386262633335313335386661646166363636323766
+64633135393937663834626134383736643139393634386465303437366563346261316534306139
-35363535353664346463336566663163303333663065613532623265303262396531303831653636
+39636431313532613464623137336334333836376465623035353166363631383733313163353838
-65353565353233626331356666343932333539356331303161303062316433633761623132333033
+32666539323465666238616331346561363938616130343934613935306533393930626532303832
-65376632376266336361363832613064323861393366313763316434316264663562616134353766
+65323762343936353834343039363332656661613139363831613366346262623732623439613366
-62643165633030363237636632386166396538666337616430323534313062333965336233333836
+65633435313336316433363339303739303531316364366164306230393230333038616465306163
-36306637323764333233666239336331373763633737623666393466376163313738393036336232
+31336231643238343964393535333936613238323339356539346464363639623665643663306363
-34613536336336663837353031323665323733313634313731326537333938396361373435366435
+63373334303235626139663331613432623539313531313937336437613763643161376462623366
-32643338346635633962346537393338653464383431396432343932373439386230613537356134
+35653166373934663935323933343733363264366630656162353164313938356431323730393130
-64386165363233636237656364396333336261613037323136363630613533353639646439303337
+63616361323264333561373062306662613033653661306364313832373336333534326136656631
-31626663393335343962663033646135333366623738346436393764353438383264666666653635
+30303364623636386432343165646535646663353436633463376534346336623632396434306134
-64643462656332653361313766656633616134373166333163346131616334343161616235633666
+61373432346434663764643435386639613562656632383962326139303233613335663637376438
-3366
+63633833393363323237616631623236653539313532663133633737373831666435363066656631
+30323766663535393735323264623330336662663039373934636531643537373333366138373864
+38623139306534303730353037373032623533313939366561653261366565313466663637653335
+33633836353966363864663961363962353061666334633165356166333731633966366239653333
+62303438613235383638383637303263623834666336393636393237313031383666336262666334
+63346439633032653338336533396532326634646236346536303862383531303430636136343235
+316337613734663564373334333963633361

group_vars/stage_prodnso/backup.yml

@@ -0,0 +1,2 @@
+backup_lvm_hcloudvol_size: 30
+backup_lvm_hcloudvol_count: 2

group_vars/stage_prodnso/keycloak.yml

@@ -0,0 +1,4 @@
+keycloak_https_whitelisted_ips:
+  - 195.200.47.243/32 # DEV-230 - sparda berlin
+  - 195.200.47.244/32 # DEV-230 - sparda berlin
+  - 92.42.192.157/32 # MOB-28 - mobene

group_vars/stage_prodnso/plain.yml

@@ -145,6 +145,7 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 shared_service_elastic_stack_logstash_01_hostname: "{{ stage }}-elastic-stack-logstash-01"
+shared_service_elastic_stack_kibana_01_hostname: "{{ stage }}-elastic-stack-kibana-01"
 
 kube_master_01_hostname: "{{ stage }}-kube-master-01.{{ domain }}"
 kube_master_02_hostname: "{{ stage }}-kube-master-02.{{ domain }}"
@@ -290,11 +291,6 @@ harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}"
 harbor_oidc_admin_username: "harbor-admin"
 harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}"
 
-postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'"
-
-connect_image_version: "8.5.47"
-iam_image_version: "latest"
-
 management_oidc_realm: "management"
 management_oidc_client_id: "smardigo"
 
@@ -313,9 +309,8 @@ keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb traefik-admin traefik-admin)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "{{ traefik_admin_password_vault }}"
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
 grafana_admin_password: "{{ grafana_admin_password_vault }}"
@@ -356,12 +351,23 @@ argocd_admin_password: "{{ argocd_admin_password_vault }}"
 argo_keycloak_client_secret: "{{ argo_keycloak_client_secret_vault }}"
 argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}"
 
+awx_admin_username: "awx-admin"
+awx_admin_password: "{{ awx_admin_password_vault }}"
+
+prometheus_admin_username: "prometheus-admin"
+prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
+
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
+
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation PRODNSO gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://prodnso-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'

group_vars/stage_prodnso/postgres.yml

@@ -0,0 +1,7 @@
+---
+
+postgres_backup_volume_count: 4
+postgres_backup_volume_size: 20
+
+postgres_pgdatadir_lvm_hcloudvol_size: 20
+postgres_pgdatadir_lvm_hcloudvol_count: 4

group_vars/stage_prodnso/prometheus.yml

@@ -0,0 +1,5 @@
+---
+prometheus_lvm_hcloudvol_size: 30
+prometheus_lvm_hcloudvol_count: 3
+
+prometheus_tsdb_rentention_time: '90d'

group_vars/stage_prodwork01/argocd.yml

@@ -0,0 +1,158 @@
+---
+
+argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}"
+k8s_argocd_helm__name: "argo-cd"
+k8s_argocd_helm__release_namespace: "argo-cd"
+
+k8s_argocd_with_keycloak: False
+
+# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+k8s_argocd_helm__release_values:
+  repoServer:
+    serviceAccount:
+      create: true
+      name: argo-cd-argocd-repo-server
+    rbac:
+    - apiGroups:
+      - ""
+      resources:
+      - secrets
+      verbs:
+      - get
+    logLevel: info
+    logFormat: json
+    env:
+      - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT
+        value: "0"
+      - name: ARGOCD_EXEC_TIMEOUT
+        value: "300s"
+      - name: XDG_CONFIG_HOME
+        value: /.config
+      - name: GNUPGHOME
+        value: /home/argocd/.gnupg
+      - name: HELM_PLUGINS
+        value: /custom-tools/helm-plugins/
+      - name: HELM_SECRETS_SOPS_PATH
+        value: /custom-tools/sops
+      - name: HELM_SECRETS_VALS_PATH
+        value: /custom-tools/vals
+      - name: HELM_SECRETS_KUBECTL_PATH
+        value: /custom-tools/kubectl
+      - name: HELM_SECRETS_CURL_PATH
+        value: /custom-tools/curl
+      # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments
+      - name: HELM_SECRETS_KEY_LOCATION_PREFIX
+        value: "/sops-gpg/"
+      - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS
+        value: "false"
+      - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH
+        value: "false"
+      - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL
+        value: "false"
+    volumes:
+      - name: custom-tools
+        emptyDir: {}
+      - name: custom-tools-helm
+        emptyDir: {}
+      - name: gnupg-home
+        emptyDir: {}
+      - name: sops-gpg
+        secret:
+          secretName: sops-gpg
+    volumeMounts:
+      - mountPath: /home/argocd/.gnupg
+        name: gnupg-home
+        subPath: .gnupg
+      - mountPath: /usr/local/bin/kustomize
+        name: custom-tools
+        subPath: kustomize
+        # Verify this matches a XDG_CONFIG_HOME=/.config env variable
+      - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops
+        name: custom-tools
+        subPath: ksops
+      - mountPath: /custom-tools/helm-plugins
+        name: custom-tools-helm
+        subPath: helm-plugins
+      - mountPath: /custom-tools/kubectl
+        name: custom-tools-helm
+        subPath: kubectl
+      - mountPath: /custom-tools/sops
+        name: custom-tools-helm
+        subPath: sops
+      - mountPath: /custom-tools/vals
+        name: custom-tools-helm
+        subPath: vals
+    initContainers:
+      - name: 1-install-ksops
+        image: viaductoss/ksops:v3.0.1
+        command: ["/bin/sh", "-c"]
+        args:
+          - echo "Installing KSOPS...";
+            mv ksops /custom-tools/;
+            mv $GOPATH/bin/kustomize /custom-tools/;
+            echo "Done.";
+        volumeMounts:
+          - mountPath: /custom-tools
+            name: custom-tools
+      - name: 2-download-tools
+        image: alpine:latest
+        command: [sh, -ec]
+        env:
+          - name: HELM_SECRETS_VERSION
+            value: "3.12.0"
+          - name: KUBECTL_VERSION
+            value: "1.24.3"
+          - name: VALS_VERSION
+            value: "0.18.0"
+          - name: SOPS_VERSION
+            value: "3.7.3"
+        args:
+          - |
+            echo "Installing helm secrets...";
+            mkdir -p /custom-tools/helm-plugins
+            wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-;
+            echo "Done.";
+
+            echo "Downloading SOPS=${SOPS_VERSION} and kubectl ...";
+            wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux
+            wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl
+            echo "Done.";
+
+            echo "Downloading vals...";
+            wget -qO- https://github.com/variantdev/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -xzf- -C /custom-tools/ vals;
+            echo "Done.";
+
+            chmod +x /custom-tools/*;
+        volumeMounts:
+          - mountPath: /custom-tools
+            name: custom-tools-helm
+      - name: 3-import-gpg-key
+        image: argoproj/argocd:v2.2.5
+        command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"]
+        env:
+          - name: GNUPGHOME
+            value: /gnupg-home/.gnupg
+        volumeMounts:
+          - mountPath: /sops-gpg
+            name: sops-gpg
+          - mountPath: /gnupg-home
+            name: gnupg-home
+  server:
+    logLevel: info
+    logFormat: json
+    config:
+      kustomize.buildOptions: "--enable-alpha-plugins"
+      helm.valuesFileSchemes: >-
+        secrets+gpg-import, secrets+gpg-import-kubernetes,
+        secrets+age-import, secrets+age-import-kubernetes,
+        secrets,secrets+literal,
+        https
+    service:
+      sessionAffinity: ClientIP
+  dex:
+    enabled: false
+  applicationSet:
+    enabled: false
+  configs:
+    secret:
+      argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}'

group_vars/stage_prodwork01/bootstrap.yml

@@ -0,0 +1,12 @@
+---
+
+argocd_bootstrap_infrastructure: True
+
+harbor_bootstrap_helm_url: "prodnso-harbor-01.smardigo.digital/infrastructure"
+harbor_bootstrap_helm_name: "infrastructure"
+harbor_bootstrap_username: "{{ harbor_bootstrap_username_vault }}"
+harbor_bootstrap_password: "{{ harbor_bootstrap_password_vault}}"
+
+gitea_bootstrap_username: "{{ gitea_bootstrap_username_vault }}"
+gitea_bootstrap_password: "{{ gitea_bootstrap_password_vault }}"
+gitea_bootstrap_url: "https://prodnso-gitea-01.smardigo.digital/argocd/prodwork01-argocd"

group_vars/stage_prodwork01/plain.yml

@@ -0,0 +1,107 @@
+---
+
+stage: "prodwork01"
+
+default_plattform_users:
+  - 'friedrich.goerz'
+  - 'sven.ketelsen'
+  - 'michael.haehnel'
+  - 'philipp.eichhorn'
+  - 'hoan.to'
+  - '{{ awx_ansible_user_name }}'
+  - '{{ gitlab_ansible_user_name }}'
+
+# TODO read configuration with hetzner rest api
+shared_service_network: "10.1.0.0/16"
+shared_service_kube_cpl_01: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-01' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_cpl_02: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-02' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_cpl_03: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-cpl-03' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_01: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-01' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_02: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-02' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+shared_service_kube_node_03: "{{ stage_server_infos
+  | selectattr('name', 'match', stage + '-kube-node-03' )
+  | map(attribute='private_ip')
+  | list
+  | first
+  | default('-') }}"
+
+shared_service_kube_ip: "{{ stage_private_ingress_loadbalancer_ip | default('-') }}"
+
+kube_cpl_01_hostname: "{{ stage }}-kube-cpl-01.{{ domain }}"
+kube_cpl_02_hostname: "{{ stage }}-kube-cpl-02.{{ domain }}"
+kube_cpl_03_hostname: "{{ stage }}-kube-cpl-03.{{ domain }}"
+kube_node_01_hostname: "{{ stage }}-kube-node-01.{{ domain }}"
+kube_node_02_hostname: "{{ stage }}-kube-node-02.{{ domain }}"
+kube_node_03_hostname: "{{ stage }}-kube-node-03.{{ domain }}"
+
+shared_service_hosts: [
+  {
+    ip: "127.0.1.1",
+    name: "{{ inventory_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_01 }}",
+    name: "{{ kube_cpl_01_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_02 }}",
+    name: "{{ kube_cpl_02_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_cpl_03 }}",
+    name: "{{ kube_cpl_03_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_01 }}",
+    name: "{{ kube_node_01_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_02 }}",
+    name: "{{ kube_node_02_hostname }}"
+  },
+  {
+    ip: "{{ shared_service_kube_node_03 }}",
+    name: "{{ kube_node_03_hostname }}"
+  }
+]
+
+netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
+netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
+
+# smardigo automation DEV gpg key
+# https://git.dev-at.de/smardigo-hetzner/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git
+gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
+
+kubernetes_with_prometheus: False
+cert_manager_dplmt: False
+kubernetes_with_certmanager: False
+kubernetes_with_extdns: False
+kubernetes_with_ingress: False
+kubernetes_with_awx: False
+kubernetes_with_gitea: False

group_vars/stage_prodwork01/vault.yml

@@ -0,0 +1,478 @@
+$ANSIBLE_VAULT;1.1;AES256
+63366239653964663337616666643236386435393335656637656461653835633030316461633930
+3836386132313735343731613062653137663034663239350a303365616166333033383231353036
+62336335306236356665373064323230626265363363643030303635623632643064336161383766
+3765333034353761300a616636643035613232306662373836613332643237373133333634326661
+37353939373631613761653536356236366534626262326162323362633064383832623434383935
+63653832393131646333653261393731323738623036306431313733356630623964643364653361
+65383039393961626233666636383539313035623431656330356261326133356235306336643231
+62313864613836356563336133333430326439653333633765396139326565343465373863346366
+32666139393863376137653730386434323533343665393861373230323936633537346437613436
+30646136356133366231306565623763393364316133636639653431653632386339343365656331
+65306438613962323062353632646630306630653133383031396336383762356266303634396464
+34323335353530323964393961373466373336363738306130343532663661373462316366333563
+34393263333634396264343137656133623638373636653636666436383862393133646265613234
+39326534613766616236656365653633363765653336343136636662663264616236653364343966
+64653164343032623061613636316464316465653066373037666664663332363938616232633366
+30623338656239373164313164363966383030326561643833316364373763303931663436643532
+39326237343537383437376261306138313234643164386237306661313536633666636139313831
+33396431393339333430616338363636383565356565323935366132316138386461303235303838
+61306432343933313630366362396336613933643635343132326431376362303634363634353463
+61346434663636373533373561623736353030633766373831306533303631343732333139353238
+38333966343262666362383363393134643965376261346136623433386462643263383438393161
+63303033666166353461646535316538336536386561353837353639323063636636623638376630
+31313265336562376637356538373138613533663037386533353833383263323439646534393663
+63393334643663373333396266383931616362363436303830376536393162326130396264663034
+63613034323134323261343633366536636463666365623437326264666261616533623338666264
+32333631666438383565366139636463623565666131306537323834313165643461663066626465
+33643064316561346437373437363832396131383637323136336330616436653430366361386439
+35333533616131643934316565366564323330313536336436353739313563633365346565653334
+38633335666633363236323939333466633332653665656133346664376436323334353964633333
+30333836633262666136343665623962626333343230303466356533663261346462633538363333
+31303334643430643332656139343332626635376365396365653862646534333938613635353134
+61383663343561336164333533643531303736333866616135326534323937643837373937323562
+32633635343736633263346332373439313637386630663231363261333432376139323462666365
+35666638653463383930663834316134396561373832333234663764653737356432343632386665
+32663062636362613262646261303866336664656236613838636336666433303938353938626333
+62393561306233336162363737646163633864653562383364396365633361313430326130346134
+37363032623233366132356266396634613136306631366162363866643235623466306336353464
+34386638336162666364643932303135613363643538353436313934323566343436643835343032
+32363866626263303564663763623835643664363230326464323066363031626562643633336534
+31303233353162303036383862396637306461383436623036383339323463666531623631363761
+34656236383031323061643036613439633832346336396239616166623532353363343732633932
+35643666383931616537336237343661363766303165306234646435336265306361663561336630
+32323438353333663464343266336433663061376266373135626431623738393562393631353163
+61396236666639656636336564383333333235346261376539313361323561623965376130623439
+32623733313532336262336466316434383333316137373934373933393362303931646566313939
+33613165666133353738616133393064613062346165303361303438393934346262333834633239
+34393131626533653465353932663730353337346463343966343765303766356233363630613034
+62333336333565313734326530643136333764313661663064656335653330343965346637373130
+30316233393965346335633137343932333664633862353566656464356534363332373961336334
+34613332363665613030643366353864656461313434643666356634343761366532363965386466
+31653962353834333564326631323662366234666231393733646536333865373266316661323861
+64623736633332363536383433666239663238366666656133613064336234303231626366623666
+64376635653761646365316564643532383861666464633366343663346536663431633534663735
+66616336613737623334376431393562336466353863353039306639353137636563633137366461
+64653630626264653363393462363533343231353433653762373038363864633966656461613839
+61373466633662306366373537613061636133323037663239383666306232336232343962633264
+63636365623965653163646339306235653730343931626363303631356631616437323934336238
+39306335303566646238633162663363386438333661303636333432623333376530396233626366
+34643130643634656233306163623361666565623565663862626434353065336432363032303765
+32613865653739623933316339353937393661313937623731643435386262373165353164383430
+39336333343664643666363137396236373061303030303837313666613863636634616364623564
+34643733636432303366386530373461333462323235323066386230343335373331343434653638
+66626136613765346430646639653338376363643964386262363665626566353933373566626166
+64393135393261333762666331666131643366343362306330666664326131333436373037346539
+38646165326635366666396237653334626166613563396533313633363866306161663138613962
+37333863633134356139313563303035383837333962313862623761643533613363626233386139
+61303363313930306462316265613863623865316437636636326262333664366665623437373833
+30326162393061313564393231386663623437616265343261333666343863363862343564326561
+32393431636437373037396533626165663938383262643166323531373635653539633933353230
+35346635363865663138666439363062333266373965383363303832396465396538343839636263
+39383564396237326664323337666464393764636534623136613735313365373664386536623061
+31373330323061346534613336313362373332333838303862633062653034623430616433393136
+38656639326231383337343764646634383063636263323138383562346130396361643065313938
+64643466643933636565346236303431613366623035633835333731356433373365656336316564
+63376566623131646232633630663762626261363639306262663633326166343366393362646335
+34376563616132626230353439366536393735393962323838663537666239333030623266393362
+66656361363563613264376661643737363661393935373766646336643534356364343932633136
+63353935643562356233393665306439303261306662643538383634363766306430643263643930
+35396531666630386436356637343330343462666430353362376539343434373538393638616333
+62386533373138363065393464616630633130653862643737373539633433393338343839613236
+64663838643164626364306138623961613630316334616563323562326265303266366433613537
+31346439393337333731333237623130363534326436616434633836306361376664376262646438
+62306136633136623135343433366539313133313637303630613937313837316166633366393337
+61666438383530636534343335626539326536613764306165383830323863393333303431663636
+33653964633530623466326437643764613936393139356162653833366538623135623364323036
+39353337356438386465303132373239663531313662356638613366343537316234633639333732
+38623934633632396633643263353130356162623538633366613237626632613165366239623930
+32613864336639616339333530396630653934323963393638303634633739393434646363383134
+32313866373961656339343232623838363866343936313761633537373666613737656335376665
+37393837333531656335616239353961343732626531616265323335313235303965666533613361
+62373962653034326365353763306336356638663839303036366462313934363131303337663435
+33353230386536306334616363306264666632643837663566373530633439613539666337343235
+64313763353165313266613961653065343139646137656330653232373338613761306164636364
+64623232346232313837303539633030356661326437336363333933303835666563313532306539
+33323162323431636534316230643936393831386137613063356364323063313065343330343462
+36656539623466346431396661343764343032373235656562383935396633373530353962383731
+36643639313862363437376466346633383737666239373435666538646561376563396265663630
+63393138376138346130346563613066323463623430326235663564336230323234623330643530
+62353634376261626238653365646561306264366132643437383161316636303136663033373865
+65666236633230643763633062356639393636303734326661343030343662616438303666343939
+64626564306233323030623363336136383734623035373761633839626331326637343338303039
+39396332623938373830336563306666646338663434383238366334666639623735623536383933
+33666331383833333965306163383233616666646537343465386361313161363466373061633161
+62663039326463343631333166616534313938626430613866386663343430313966646262353465
+35353262663461353362626264643036326532363535366166303933663335653634616361303062
+35356535393931636365353861613062313334613663626230396130666462386464623663333832
+34663833303737656465646362316137346263313935626237376561306266373932663233643733
+61313262383261373763616532376532356636636635663237363561323135623263373330623630
+36376136633266376432383265663862313964323435366137356162373132386636386234323836
+35626231356230646637653238306539373466613737316437666232643265633836343461623862
+30313164643538353630623363333030626166633638633666386536633736366663353261623738
+61383536393631343734366566613764613235616430386332303438656136353230636232623836
+66383234366330363563623932366139323933643033653464353735623037656564323434626232
+39613836306632323763356535646234373332313161653233316237333334356131356438313533
+63643464663937393632383166336130383864386335316530663066376662336261653465633339
+33346537343033336332383537633466333966646330383561313234386238643335333966653662
+30333234643933383433663662633838323963363863343665383734343932616138613834633762
+37636564376237633331623766376434303735663865653565653038383363333964373736343236
+30653662353932623732316534386130633131663539383465333338356130376334623161666465
+65636234323639323030303262306565356166613834363132366231343430313666303336646261
+62373339353331313630636264643462643736323739333966633035653935366231333438646262
+36366236663264663431383237346562353335663134646134653931633933613732633263333635
+33363265363130386636653835393939396337386336646437323138366637343535633337663964
+62333133356266306164353836333133653862643364653537663734623239373330366330373032
+34363163313237333837313037636162663137353937306164313937316433336630616265353665
+65346462396135323862303764343133393338613164663733646239373737653037323939373661
+63326335343162663763323638663033636534366262333664373931323333316236616339323733
+61303761386563343731636465613937346661396531643066316263356165323564303961386533
+37636637663366626630663165386264316537346166653865616335376564636266323033373664
+64366337373234313731306630653136643932653738323433363232343536346164623933333639
+34326566356434396334373665313233303164623437633536373833343332326265646539653734
+61653366353365643262353936303764326561313731393366353731646630646536323164326163
+39666439663736323835386234613738333962343162643532386632666235373166363938306635
+36313938356338623363653037653036623634656435313935303831306439343232306233373861
+32356464646362366266303763636439646563353836376261376334303462376139396335366135
+37363761366265336331366262376433633036343238633736393130303234313664643439393734
+33373936303535643432316230333532663963366263333461306534326433306661336639373961
+36653933373432656630333463636639663261303465313536633633633636353331356633396339
+61306562356432323265313466643932326564306462343363646334323363663065373937346265
+30373731313266633537313862356535323533323465666537366131346538353166396437336534
+62376131623062393965376166646337346562626634313265393665386230353435346336363736
+32313234313634653436363532306139643933376538616166346533333164656139323561326235
+33346439343939346634383263666531393432643266373731663331313435656439356362653066
+30653237326435363238346130393533373465373232333562303361663066383064393238643434
+32333236346636646532376266613236323362623165306561393462323337633831636561663731
+36323664616664343261613062306137646664633464663439353465316662323365366631653765
+64613864313931643136396637393337663665633931366662353839626331306630306533313462
+36343333383835633066346465383730306432303737373933373564326137353738646366646262
+61663131663132626532616431326636393037326133616139336565663432613234613838623738
+36333131636439353561323238633564663838653936363932303739633331366366643339393638
+31303364333433386431313363656364303337636530306264303532666135356636666466663234
+65623438626532393530666139636437353838633134366662333531366132376562373339663465
+34386536313866333832326437626663666333313866643432633837323862666233343936346334
+36653561373931623532316136386363643762333438313034303539323734393131376536303832
+64623463386237633463633431653766343032653132336331636439306135333362653239343562
+33363861616336666463646331613165623539363131653666363364336631356639613264343837
+66623430316138373338653739336637333565643336313963393934626330386235653239386564
+61336131616464313231616363353232623465396336623230396263386465376333323134303864
+66653637363665323632333762383536613533616132323435376132316361646531343465663564
+32343436376364633237383964326530303034393161653438646131626135346165653862653433
+64636166373266386130393236646238663066646337623562336461343136656433363931393063
+65386633313462393339316238333535663233663632653337353333623164313963366263626433
+31303363336433663461373064383562646262353733653633343536633264623663376632306462
+37363264656535313332613666616564303864633262313366363964366231663638303166666465
+65376136343563656234656366613932616235336337353332326362323931303236393638616132
+61343135393533323130633435626665303331323830643739646261666636613330623763316537
+30663834316464376433643163326330663632353866353038373032643930623032366264313430
+30613566363861336134356562336439313333666333643961623465336532373138316363326334
+35333863326337353930636235623932663533396464323764333266313839623437663535363062
+39376638383935373939393336376338386362336439613236633735633633623166346234353664
+38633664393634646361626564336638646162363066653464353932393830613136303864616333
+66636437666139326130393436396536316630623132326333326539333935356133626662313034
+37353534393264383634663362343062346634393438346364613464313564386436626433613462
+39323133613832343430386433303034663534366334643636663234326564346331626566356364
+39366632333064633836353838306230333964633336376230616461333731386631323836383261
+36383034366135626264653539643633306631666630623634383834343734626130343064616335
+61303032386636383237326230653665616365383861396161313434306139363439636137356236
+30343666373837376134653936616137616465333139643336393866373436613262633364323233
+66363237333161396466373264626433356464376665636366656532306461623431663538356537
+65666366353639383962653334396533373065343537353535633933663165303338643638393130
+30366632356363376332303866616162303161326430653466333066303461386131353333306164
+62636434623638633834343865376133313835363933323266353936356563663537633161386464
+35333364656631333833323630313332353430656531373035353331373936396261613038656262
+65356330373431313633653464336637663362333762326162386562383839623039393163343761
+65386261323136363732626461356137316563646435663136663334303463376436363563303234
+31383165346362356665623634653537353530386438666230333166626363353062383137323833
+31303166343330333733653633353930386263383732316664643134323033336230663333343863
+39343266663865346239653639343834613536316233653137393037666131663666303530356239
+36623663313135653466336637323033376539663963333566613439376530306332666636613761
+61623663353636396531326562643231373433316530386433643330343036303164333536656639
+66346362616638323938313366626237346332643962393939613034303735663130343734633236
+65396361396464323939323861633762623961613239623832313630363661613664303734373038
+66316164623231333730376662346133306536376562616639636139323866373439363036626464
+39313331663466376661333239646565326438396161353439356431613333366335356236383034
+62363038616231626331653934666338663537306335303731633532313233333431643239623862
+30663563636133646133623432663862613232363862623062386165363062663666373936616164
+35616162373263313236666437353364373561353234666662346337396661643261323033663939
+37663039353231613366306432343638383037656230666531616639396165623963613732333638
+33353933303964666133343136333730633131663164333835363966386335313035623764396433
+34313737663335343464643265316434623863313835363832646133396365666562343534396134
+39383134393535303734616237646561623732323637663032373361303963353532613762633437
+33396230386365373961333333643931386131363866366633306432316231623132333366333137
+38333730316533623266313261306365653561653436663464616166653031646664633564303037
+64376538333563396365386461643864373436353163653738343538656132363063373138373834
+30313038396363663032313932636266643866363462333137313935373339653163356265323537
+37393764316432366537336366303963616164313865333333656536326435326263623134646663
+64323036626635616166366138623030396265656534623263373730313235316163396438623062
+66623630336161373932336263376161346438663462393234613039356265323636326131386330
+66656532303363303038323238386438346463633035373665346434353239616531393664646161
+61326437306463623136383763393933396638613032343934333964303162616666653363333331
+33363436393531376530313464386563616539343536663334356438323131306131626331303438
+37306166666165356563623230393264396533333063313964316462613461643633376338633439
+35393630363632363539353436656564356562303138616230383961396163316439316335613538
+38666238663930376166316633353138636334613665613338373165663937643961663738613734
+37646637613335633362383231616532323132653932656337373134623962633464393730323264
+39323438396366343531623263626534363061363163663031373137663132336633343061613065
+35663464396531363865666561643765343032323066313533373831656564303035636665333162
+64663062303035333933373730633033613861383961316330656330643637623534636432353836
+32383432636461613762623935366362363664343261333762643139396333613039393635326536
+32356564666630366630613361666235336263336362623930333366303030616565353933616666
+62343466343434633434313562353162316261646431653530313262376465646662343663386637
+31653865313763656635393831343665336139653636643361333835353930336533383462333561
+35643530636165613435393263303464643133623565336131323832343636393130616633623666
+62386663653537636637626531333465383630616462393433376436356661373231613164643465
+61393332663666346535643963306434323965373339653731373564333963373331373834636330
+35353764613638363333636138356436336561666265356331353163623732303633336630633462
+31346363336433336437643331656566363930646630666230366332653234613933393064313566
+62646266373035313533326264636237353163383761383066396437633965646634646136386335
+33323664626332616362636461373030343434313062303831653838343235353436303031626165
+38333733326534373166323832613437393739306637306566356566383139663666656633656531
+36333830636266626537636436303762386363643739383161646538653039356365386536653437
+32303030346535613063653836336338333535343437373566343335636464386231303834646638
+65356437373365343735626365653235663636306433346632313663636465343730633233373461
+64326465393933323530323862313261656464346431636265393464316638653835303834333365
+36323933623533646230346663636631303066306430383764386632326366313038316362343464
+32313231613166616231393962653862396565633134643133393037366565396537353339336437
+32373131313031636439646362353234303263646265373639643638336338646537373735393661
+65623432306633356262656331333532336539353439653362643561333936363961366233616130
+35386232613538386230636431643131653062316463373935626131366364316534313237356634
+37306639346230653933396462623263613630323566363566626636343139323265323633363235
+62646333633339396439316365326162376634343861613935626431666162383662396430356231
+63356337636136653436343832356131313162306435386165356163333431616263303334313331
+34353638633536336166633865386466373433396136303562666537383462333535666338316430
+35376432376431353662646437316634626536646131626463613838316234366633616235666135
+39386631383562343765613162633231333061336663643235363931643036316564303365383334
+32323530663131323637643733653635613334316265333734333930626635616562383135323830
+36613330666431393232386539323032616164636532663862326265333763313263313363383634
+30613533326438666334323439666339336232323234396439653039336431356635656363356466
+39626530353132376334336534616464323535613731613235343437623561326163376537326631
+32383838376230323364346463653037343162373833306239383163303439623763393831303733
+30303165656364646537336364663330393664326339323234336139383330636162363930366432
+30393532353635396337636434303066383236643431313362653538316263363135336332633636
+61333832356536303534343861323166333264316639343564616565613865363765373130363231
+36636339633733363630343238336132363664623661633863343635393834313966363539313563
+37633231643864353162363937343836373265333734383736343366366163396337303063333764
+64313537333065323935643462616264613430353466383233313932383634376333393032616663
+32373131366230653831303962643630306630343231386139623965336537333033383039373462
+38623236323336613034333234613761626666633332356330666439373736623165353163353764
+35323837383638313562376433386561323136393631396236353166303331643761313064643535
+36663130666539663330376665333335383464653264663339363836326464393134396264643164
+65643962663766633066346634353234363634326434653730643464646338613663636235666562
+64346636303237353638363435613263326133326162346562633864633765313533363432363361
+30653235356438616166383961633838336338303735353733633063613238643131663839623131
+37373835386431623364356336363735343834313132313531613534313238353133353834623835
+30396466623136386537376635306464316530363334666535646461333036386232333937616666
+32353130353364663337356333336536653238363662356263656434656366633664643334356432
+39373037346433653463636231656264373538346432613731616136613065396434323237383863
+30313231373931353235626633613930363936363733366131633063623036313635663336643761
+31623165643062333832333837613435373935313562656332633032363830326664633962343061
+30663633396439333937666139326530356437383963623031333366613964336435376137363736
+38623239366632373066393834333935633661636335383531643237633632356339336130353161
+62323462653233623836376431336661623361343062613835363130663366383835623966346239
+66396262633561376635366636666464646138626236356535663238323732643165396133613561
+66303862666531396530666562626632303235653762366463366261656665303163616163353139
+63613436623935643336643538386264326162373037633237623665393634623332643438336464
+35666539356636666531623764383339613730346666353961323235316637333837313230623836
+61636265646663663064636636643538366337643933323362613263646565376439316436613031
+33373936313661393732306636343932636339393161653931306137636563393731353639663130
+61303330626563383062343662356639663636353462396661303032616265643364383239613037
+61666563313766323536316237666666333237383263353438393935373032313734366433316237
+63616362323835666138323066373939633763646531323362386161346532313939303063343731
+37353538383161313966333932303538656339613731323064386131643739663562373537333137
+66303238633061323864626662313130626331656132626636626630656465376130653433373531
+31653436623836313237303930636164333733373861326564633930366536383337643833383136
+62626331613234303933393165663836346261353965333966613534316438313439643839313434
+37636230326331333737623937353134363633363363303163373932356230373431303137323034
+61396539653762633964616230376334366665303765323837373638613332306338653662386261
+33343636633936633666643636613961336665636163656263353634393764386134306662653036
+33646637663561363330653962323264653130343333653231386361393363656136333335656532
+66336237396263653063623133643966626365633866356462653261356636383661353863353936
+31373338313465363837653036313333623737653365393334353861613663373730306363316635
+31366634313633303865376337383565373931376562313461653937363736343837373663666536
+64316236663666363061373139363836646138666462363033613936313735383230663336653965
+36323630336431646338373837393665646265396465633965353931633935386338636631326131
+33323739313536623437343831666362383733636262656266386566656436646435373432666666
+62303230326464383538373761323461343232653139653935653833353238323836646337363266
+62653933343038336439303334373635653662346230623530326637353433373030643531393239
+31353561653665346231636134333731646639663931656139666131646435343661316231353134
+61326338393438663930336336623466326562646261626461623431363333303161383039303835
+31353465636561613135373736343164643739636234353764393735653730356266313737636364
+38366633396438363561333838373361313662386337386331386265336535623736346332356438
+62393936303061366636643037663131346364363462333335343961656361396562616461356337
+34343031653364356636386664643265343363306562333233656234663535313130326663383263
+32326339643730393632393234646535643832306365303162626435623538376461383133336661
+34616334343037623839396334363739383238633761653565383834366465653432353339623131
+66633539393564643731303435393038316332336335386534353139386138313131356165623763
+32393439616438393164313163306531386139373431613539356138383033623830643932383636
+66353736336332643866306365336265353033376361363362656165656664636164383639303432
+64613235613034336236663833623339623865333538373963306139643439393466306438373035
+35336162373336656331643930633032623263373264653565346532363239633339353634376132
+38353930633765373362663137613136313566376134333533613234666336613461653161396431
+38636638663461393839313262616333316236323864616439346138373263366163656537303964
+63376161653037653030616236623837653634336166323139623733386537306333336661633439
+65316432323435616133653536383235376165376437633836396165613137396434656164376136
+35366638646637666130393438313436336266643761376430373730623337396634306262336564
+64643964633739333962616433636233353837633766326239303066346337626132366166626534
+61383462383562636535323832623930306436333435613364343637633938616438666332646631
+65353561666630326439656461653763336362653736653230633330356566646262303839333834
+36663062626430333738666136383037666132303231326165303865383365646461356164383966
+64316537663132333061626332363637383534356639353039633966633339633131313036633732
+65356361336539363739396635303337373763336531613861623431313430316132393034353036
+34313732626665363737303663616633386231346430613661336335383036626230666332656436
+33666263356364323364636437393266653966613837613333393065343231343632333964333535
+34343564613966613630333830343530326364346462363639393231303463326338346536633538
+65366533616431636565393536343637393035306438643236363832336439323932376263393362
+65383361643530623761663036303632656561396663303362316465323130386435326666653738
+33636364346362643733633764636363396636393664396330393437366564333062613432303266
+62626234643634363234663736386336346164323962663363663965393565653936613836326233
+30643030363961626463323463326563343164366637343966616637386162323537636465656339
+30373337316130336565643332633664353136336662613766343931666637326665356235343838
+38316134316536653261646164633863306639356337623261616533366465653439383464363537
+30636338353530323533303734613232323334393935616438363533303939636235363135383838
+66626534653531633035333233303663633232386639373932306335306264393033623562333566
+65623033393365386562653037636165326432633931393731626131393530376137336562336664
+34306338383265313034623033643137393962303964633839653233383362396134313839316265
+30333734303066626263666566353433656235373239656336666261653736313532663164356233
+38393231353338633966353839653933383239303631376335643761633362363030653930346134
+64346161373963303236323130633039653366646232343866623039316430646339323165393266
+33323730643664653233643661303762616137366563366333383661616466316531313462656362
+63333939653863376333316563316232303836346130653630336530653031313536663835326262
+31323338323433333866346366353066333431633966663934306466343633376364646161633031
+62336635323663353438623763663138333164653563316461333764653639376438393230643730
+33303366613434363134376432326563393531306564313035636438313762363433343666363764
+36663837316666613738366631623530626434643936373531616134626338383637623663376238
+39623861333165316438626261396630643738613962663961356231333663393062363935376335
+30313732386330313339356437333230396462336665366139353466386635623564383066323666
+37626338353563386436623061633630343138633136303837356335393461316461373739333736
+33316538373836313764356637636136663334626264633134323134383430333564306238353130
+34643036623862666133373061343031613734323564316163616235643632346436316536303032
+38313231353832356265376433653137613730343465336566633139653339303463656234613831
+66643664333937393866373837663236663231653232616436326432376163343064326435663766
+65346662633739323065373234653263323531626633386466633030663661643837623534303832
+38343535636666656137343965643661623832666531633163653039323933393038623832333636
+35336139323830663331343030643864363631363563373036316237386661643861616134336638
+63333736633439303863633164656366363861643438393836373865386131376666323761633564
+63306236633034343630386437613231353731366530626138633662323230366366626139313466
+36353138626330336134636232623537363165663562303637666333306666633631363536366262
+62333539656232353432353761663837656364363435653333363535306336323033653137643864
+64396333366335383530303835666563373033346464646637306562396134366265343136613534
+38376137303563383733663565616230623364366635333062336132316562306365313937396638
+39393165366137343866366131366530616564626437623265383039346235376130313161663638
+30366362613532353461316634383533643335666365373966623435366433343836356562616635
+39333161316636386230393538343236613732613432646237333035336561313833363966323161
+62366339376166303766356136633637306561323231346164303061356338376431356264323264
+62623337636364333439373439643238663630653232366634623830616463363163353765653032
+31343236646430343937613535663639363465653439653337383761396533633337316562666662
+39626330613437323063313832353731643031363835663762653136366435373334666361363363
+38316134313339363737306233333234363133353937386262383431356262623332326130626338
+38623939623139383032303339306435363033643730613733373666343765643034363231613861
+65336539383839363662316564303935623234316636626663616437376662346164346266343032
+66666233363964363638316166363739633533383335313732346237356562623862353962363035
+31653131376263633266666530643434666638666133383731613861313836636261663930366334
+30336131376236336532366634633437626139633130646430333538326464373234393864393262
+62393639616239613762343663663731323933363365633332616439303865383530323461663962
+32623364313266626637373164656632623931313563386536313533333639366162656566626232
+35623434393136333331326362653566376365333466326532383764613431383334626565393138
+36623564353436643666373832303838666434393265643630376632666335396661663732633162
+30343163376337623739613530396537356363623562313732643836343032383636323733623839
+30346130343736343537666130343335666663386664383831313631323535663934663931303230
+63303833326466386331613165643763316266323762366263633066393161623738373838656164
+64633934353036313662633965366463643039363235306462386635653432353334316363313435
+37366335616163393962303031396362346234623761343262653636656163636131613034326131
+35646636363539616530303865353339613665396237616338663434613833663634383733383862
+35643334343934376266356635636530333063316635613135653138613266656430363432386464
+30313864666533336336393931626432623834623662353266666138366435633661653533373133
+30613562373234326536393433633131366434353635313233333131613661323862616339376464
+61343839623864636439373733653437373133313133646130353032666466333237663363353265
+62636539633163643038313039323935316665643839663362613966633938383561323735313638
+35633164646464363565373065333865366666653937336231613836613363616564383337396363
+38356132666235633361326262353931643732646333343933666533653539363039666436663061
+30386234656661376432356135383232663232363764356136393033643233343438626464346231
+36333165613262316432343838666434353363636133633437666563323138376261316564396161
+37306134633565643565383938343664383137336632356432336238623339613334323736653165
+36376634353532333331353362636462326462306564656331636632386338323739303136646335
+38626437323030656632646138333962353636383531366138653566393464306230623963363366
+36613964343835326264303863383530613237396632326565386165613431623930633930633030
+65613130643333323632643031343433393263636333353861626139366233666465353763303137
+35623336306165653737663239653836333861323261363832323834623632306462363866353362
+35323034346337613633306162363063393665306236666433356539653961623638303532636262
+61633235333933303138656538303237636566623239653461643362643532353062353839383331
+35316235376662336631306532656235643435353130393836616137323737653633613136313731
+63623030306661636130326565646636643361356632336635636634363530346561393666346133
+38313961616338376666626333633162336430303231613735613861303539653932653861393666
+33353761616631386363656436623866323565316562643564386338656466313731646365343138
+39363237323765393834666431353065386538613063383965626363303232653235323139346433
+37306332633836303039623166313662636239343063663034613765326433393138653936626438
+35363266613437323939326165626635306434636637363638616338383434643630383837356462
+65383032663033326233303163316133666233663031643131643061643164623835306137393262
+62626361383561333165663538666335653730643032363262366533666565666537643262633337
+64636665636364373433666530616638356232376366623364633432623461353564633833313665
+32316636376539663966626464303531386263336533386265373534663939623661373162633539
+34313832646331363665633333396337633065313264373137613764316339373964636564383066
+37636661303962373164663665666432643437653537336165323938356630616637616130346331
+35633736323433656632666664386561383066343434303833323330303136333461613730353630
+35323136376433666465303438373937326639323561643636633964663434636465343563666630
+32653634613732626363316532616437396539346232643834303265353839326139313262373866
+64343062363035646237323138646639333633346466306261373366373163626364646265366237
+36306265653134353639376466636537363534373034363461323235613530326339376238376231
+32383337326564323435613533383632353266356331366138346534376462323062303030336466
+65363235666433326161633662333631366131633731356166656431623232656361623263663533
+64383734316434313364333539336263373135646566363432383037393366396230653038653161
+38383536373064363232373135646638393163323935316435343166396631356137653531613831
+34623065313165643765376332343832333161653639366139343434353630393239663534373634
+34646363373466353165363832323833613032656533343761363162336665306239363261643961
+38386464613561616237393363353934336436313036326266376131626665313836333165623837
+63616436353431623630303961386430656136313733326264386466363333373637306130633530
+30656666666265396537663233343732643139633636663631333139373632363832306166366330
+32313437393231613837383237383534383835636163373536336665376462656331343038353837
+33323063326564356662376130343364353863633936313630663230343461313136386536663662
+63346237306137656431376364623531366236653137363966656266643135326636653934373662
+36633664393062643863393865613633646435336138346432363534613436303462303930666337
+30626166386264363236626135393438363938656362653237623463383730316530646238643264
+30323963383332633531393539333637323663376338646662326563383861336632376134353634
+65323637663139326363633265353462613431326166323732336266626333623064306436326434
+66616563373735323138663763656439346333356632323161373563646632393534356263633837
+39646466383430666566316639386661323135613461343563386366303265373536383961343038
+35646636393562353238613839326134336439643564366561373530353964353937316363306139
+37616234363831376136656564303430336662366562393461383138336639346364633833623864
+34313133313363373433636339643537393638653031633266663363306536366261656631393038
+38393239396633386537663437353462313137393663396137363462646236633665366631313534
+34303135316332623437396633356230363963316264636137383837666635633266636337323335
+30353238313739323738653065636533313662663936616337313739346363393530373465333564
+63363138346435386564613339366363626565613365346535653963366462303039313138356463
+63643836626138323761396535313932643263373430326336383165373465366339376231626539
+38616631333430313533323132376266623664663065373936323833303461653435353862393834
+34613433313732326638323465653535633861663638623730393730653861373233633335363166
+32373439393761313762333131356635383663323638616562326636616530383063386665656637
+38343563353636316464316137663638303630613930386462636639363839356231323164666161
+31346235383539366562666639366466613034336130363164366464316666633135346462386430
+35353862656436336461653136313231666266336131633762353030663865326465666235383864
+37323731353661623765333834656331633465353866303734376334633537303438393532613830
+61303832623239313135643437343231663136643738636232356565633165326562326136643132
+66356462336466306138613033656330666133383763656135303065336638346537353133396435
+36363264393866653831393636396134336138316364356661303235656339363266343434333336
+39336233313832623632646131383435336439303534356237646663613438396430643361613932
+30626538376439626236313233643363353365393861346536626335646237343538363736396563
+38373962353763313930633830643231303436366636323438366431326532383265623264343732
+37313632353863643363653137363361343437643262373433666234366638316163316235353838
+62393134373034623136333037306130623030393439623337613933323063343165343635393830
+38373130363731313931333535346133613234306535336162653630336462373361636664346266
+66303937326535393439366233383632636564623862383061383334623033616138656539303138
+31366361333132613630373635633839613761363566646232653936396664303564613631616434
+61633737326166343263366137306537653439333036666636376166643436613766373730386631
+61313237336236643963646562666263656234333738313938636335353837393339633361366364
+32653230313564373062613863306633316461643437363861383763323039646232356564356135
+36333236333036656430333134363130353739613739343535306462333966343764356336613966
+30336537663437326535663130643561633539333863326239643063643661626438393632396661
+65343462643433636165303634316539303431366430376665626663336633633030393531363233
+34373438643134613166363839613931646363396431373734613036363534343832623638663938
+63636365383533323731303332633433343137393265366135376562333137363538653963336465
+37633630636638376264373962396664366635383336633832626265353436613834663835363735
+34626630616461386133656366626466396364643737633938343130663836373935386333656666
+64383134386130646137373039666531663637376636303666393930653164613139626163306666
+37653463633736653831303964393962353165323765363436326261303163616434323161353231
+66326639393965633139636563323336313330313331333562373761313538343335613036626461
+61376336373838336363336161353633663132636331613537376239376131353234336664663837
+3430

group_vars/stage_qa/plain.yml

@@ -145,6 +145,7 @@ shared_service_elastic_stack_01_hostname: "{{ stage }}-elastic-stack-elastic-01"
 shared_service_elastic_stack_02_hostname: "{{ stage }}-elastic-stack-elastic-02"
 shared_service_elastic_stack_03_hostname: "{{ stage }}-elastic-stack-elastic-03"
 shared_service_elastic_stack_logstash_01_hostname: "{{ stage }}-elastic-stack-logstash-01"
+shared_service_elastic_stack_kibana_01_hostname: "{{ stage }}-elastic-stack-kibana-01"
 
 kube_master_01_hostname: "{{ stage }}-kube-master-01.{{ domain }}"
 kube_master_02_hostname: "{{ stage }}-kube-master-02.{{ domain }}"
@@ -290,11 +291,6 @@ harbor_oidc_client_secret: "{{ docker_registry_oidc_client_secret_vault }}"
 harbor_oidc_admin_username: "harbor-admin"
 harbor_oidc_admin_password: "{{ harbor_oidc_admin_password_vault }}"
 
-postgres_listen_addresses: "listen_addresses = 'localhost,{{ stage_server_ip }},{{ stage_private_server_ip }}'"
-
-connect_image_version: "8.5.47"
-iam_image_version: "latest"
-
 management_oidc_realm: "management"
 management_oidc_client_id: "smardigo"
 
@@ -313,9 +309,8 @@ keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
 # Note: all dollar signs in the hash need to be doubled for escaping.
 # To create user:password pair, it's possible to use this command:
 # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g
-# TODO should be part of the automation (htpasswd -nb traefik-admin traefik-admin)
 traefik_admin_username: "traefik-admin"
-traefik_admin_password: "{{ traefik_admin_password_vault }}"
+traefik_admin_password_htpasswd: "{{ traefik_admin_password_htpasswd_vault }}"
 
 grafana_admin_username: "grafana-admin"
 grafana_admin_password: "{{ grafana_admin_password_vault }}"
@@ -356,19 +351,23 @@ argocd_admin_password: "{{ argocd_admin_password_vault }}"
 argo_keycloak_client_secret: "{{ argo_keycloak_client_secret_vault }}"
 argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}"
 
+awx_admin_username: "awx-admin"
+awx_admin_password: "{{ awx_admin_password_vault }}"
+
+prometheus_admin_username: "prometheus-admin"
+prometheus_admin_password: "{{ prometheus_admin_password_vault }}"
+prometheus_admin_password_htpasswd: "{{ prometheus_admin_password_htpasswd_vault }}"
+
+alertmanager_admin_username: "alertmanager-admin"
+alertmanager_admin_password: "{{ alertmanager_admin_password_vault }}"
+alertmanager_admin_password_htpasswd: "{{ alertmanager_admin_password_htpasswd_vault }}"
+
 netgo_msteams_hook_cd: "{{ netgo_msteams_hook_cd_vault }}"
 netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}"
 
 management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}"
 
-# smardigo automation QA gpg key
+# smardigo automation {{ stage }} gpg key
 # https://git.dev-at.de/smardigo-hetzner/communication-keys/
-# push mirror: https://qa-gitea-01.smardigo.digital/gitea-admin/communication-keys/
+# push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/
 gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}'
-
-iam_opentracing_jaeger_enabled: true
-iam_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-webdav_opentracing_jaeger_enabled: true
-webdav_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"
-connect_opentracing_jaeger_enabled: true
-connect_opentracing_jaeger_http_sender_url: "http://{{ shared_service_kube_jaeger_collector_hostname }}/api/traces"

group_vars/ubuntu_docker/plain.yml

@@ -0,0 +1,8 @@
+---
+
+dns: hetzner
+domain: "smardigo.dev"
+traefik_letsencrypt_provider: "hetzner"
+
+hetzner_server_type: cpx21
+hetzner_server_labels: "stage={{ stage }} service=ubuntu_docker"

hcloud_firewall.yml

@@ -47,11 +47,6 @@
       loop: "{{ hcloud_firewall_objects }}"
       loop_control:
         loop_var: firewall_object
-
-# set ENVvar awx_related=True to trigger playbook part
-#
-# needs to be implemented via switch due to potentially missing nodes at first time
-# when playbook was executed
 #
     - name: "Generate awx-related hcloud firewall rules"
       block:
@@ -76,21 +71,59 @@
           name: hcloud
           tasks_from: configure-firewall2
         vars:
-          src_ips: '{{ k8s_worker_node_ips }}'
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
         loop: "{{ hcloud_firewall_objects_awx }}"
         loop_control:
           loop_var: firewall_object
-      when:
-        - awx_related is defined
-        - awx_related
 
-    - name: "Setup hcloud firewalls for database backup stuff..."
+      - name: "Setup hcloud firewalls for database backup..."
         include_role:
           name: hcloud
           tasks_from: configure-firewall2
+        vars:
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
         loop: "{{ hcloud_firewall_objects_backup }}"
         loop_control:
           loop_var: firewall_object
-      when:
+  
-          - backup_related is defined
+      - name: "Setup hcloud firewalls for gitea..."
-          - backup_related
+        include_role:
+          name: hcloud
+          tasks_from: configure-firewall2
+        vars:
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
+        loop: "{{ hcloud_firewall_objects_gitea }}"
+        loop_control:
+          loop_var: firewall_object
+  
+      - name: "Setup hcloud firewalls for keycloak..."
+        include_role:
+          name: hcloud
+          tasks_from: configure-firewall2
+        vars:
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
+        loop: "{{ hcloud_firewall_objects_keycloak }}"
+        loop_control:
+          loop_var: firewall_object
+  
+      - name: "Setup hcloud firewalls for kibana..."
+        include_role:
+          name: hcloud
+          tasks_from: configure-firewall2
+        vars:
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
+        loop: "{{ hcloud_firewall_objects_kibana }}"
+        loop_control:
+          loop_var: firewall_object
+  
+      - name: "Setup hcloud firewalls for management..."
+        include_role:
+          name: hcloud
+          tasks_from: configure-firewall2
+        vars:
+          awx_source_ips: '{{ k8s_worker_node_ips }}'
+        loop: "{{ hcloud_firewall_objects_management }}"
+        loop_control:
+          loop_var: firewall_object
+      # end of BLOCK
+      when: hcloud_firewall_app_specific_stuff | default(True)

host_vars/dev-blackbox-01.yml

@@ -0,0 +1,7 @@
+---
+docker_enabled: false
+traefik_enabled: false
+filebeat_enabled: false
+metricbeat_enabled: false
+
+monitor_port_system: 9100

host_vars/devscr-kube-node-04.yml

@@ -0,0 +1,2 @@
+---
+hetzner_server_type: cpx41

host_vars/devscr-kube-node-05.yml

@@ -0,0 +1,2 @@
+---
+hetzner_server_type: cpx41

host_vars/devscr-kube-node-06.yml

@@ -0,0 +1,2 @@
+---
+hetzner_server_type: cpx41

host_vars/ext-bdev-demo01-01.yml

@@ -1,5 +1,5 @@
 ---
 
-hetzner_server_labels: "stage={{ stage }} service=connect tenant=bdev"
+hetzner_server_labels: "stage={{ stage }} service=connect tenant={{ tenant }}"
 
-hetzner_server_type: cx31
+hetzner_server_type: 'cpx21'

host_vars/ext-bdev-mpmexec-01.yml

@@ -0,0 +1,5 @@
+---
+
+hetzner_server_labels: "stage={{ stage }} service=connect tenant={{ tenant }}"
+
+hetzner_server_type: 'cpx21'

host_vars/ext-bdev-mpmexec-02/plain.yml

@@ -0,0 +1,12 @@
+---
+
+hetzner_server_labels: "stage={{ stage }} service=connect_simple tenant={{ tenant }}"
+
+hetzner_server_type: 'cpx31'
+
+connect_external_domain: "ext-bdev-mpmexec-connect"
+keycloak_external_domain: "ext-bdev-mpmexec-keycloak"
+
+traefik_dns_01_challenge: false
+
+sma_jwt_secret: "{{ sma_jwt_secret_vault }}"

host_vars/ext-bdev-mpmexec-02/vault.yml

@@ -0,0 +1,22 @@
+$ANSIBLE_VAULT;1.1;AES256
+62396561376535633331366135626166313361653939363663623337353163353932303265656338
+6133636136653233363037323831636662653238356132610a356535373932336439376132353139
+39343134613366623462313361326230316338613235306539306330313861393137386137353035
+3430643161656532310a363565656236306238306661663864396265343162333136313130333764
+32643366626630323466386330656232316261373130363035306633616566333638633234636634
+31373866616363306663313165346532646464313065326637656630363335663663663164626366
+31623133343637346338633162366136363661333339623761313132336437393836663564636137
+39346133643439376230346439356232616363613839346664353761306535386331333766313334
+37613030376339653130386264383831643539323866333666663338336366343266666231653761
+36316633356537363662656138626135343636666635613264663339393632643362343463643537
+62383931356630303863323039326664353235613635353164383530333066316430353638663263
+36353733393337626565373435306134353764363230656662653538626133303332633034323633
+38653238646562636265393862666432306235663862356238393539376563626438313335613933
+64343636623938356433306265346233643161623131356238386162353466333330343930376265
+36323537633566363364623164343938396165396265633763313434386438356533656430313931
+62313836326130343261346435383137653431356335383633326162646566333964643132383065
+63343030663938646531346638303433623435323662616333613861626133356531356135343334
+65393663306364346432636133376335343432363664343263363439363434643266356634393132
+37396565343430306534363238313561643032383062303833353732303739373030336362363234
+65373366636662313330613163326265333933356139663439666634393937356133623235663265
+3436

host_vars/prodnso-mobene-cusprod-01/plain.yml

@@ -0,0 +1,14 @@
+---
+
+wordpress_image_version: latest
+
+connect_mail_protocol: "smtp"
+connect_mail_host: "smtp.office365.com"
+connect_mail_port: "587"
+connect_mail_user: "{{ connect_mail_user_vault }}"
+connect_mail_password: "{{ connect_mail_password_vault }}"
+connect_mail_properties_sender: "Info@egeld24.de"
+connect_mail_properties_sender_alias: "noreply"
+connect_mail_properties_smtp_auth: "true"
+connect_mail_properties_smtp_starttls_enable: "true"
+connect_mail_properties_smtp_starttls_required: "true"

host_vars/prodnso-mobene-cusprod-01/vault.yml

@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+63313634313235623162373139646237316436336364376237333463303339636135303036323135
+3339326265343539663634353235306436383963666162370a313862376337663239663162396163
+38636336646465636339353032636161613034363434346436326364653165323632303666323464
+3162336233343635380a626664376232653734316334383561333963343266616163356430653361
+32353934613365303464653938626536656337363039326237633835643662653032363633653263
+62333935353365653039383638353266633632656638346332633563323566306532336538336462
+62386634323937626662313964313933616336323935616231623637363663626231356533303063
+30326266363334643431336233376462303637303863656138333763633361346335643533336134
+36363231376638376433353061343334356238313464343266396537663630363430

host_vars/prodnso-mobene-cusqa-01/plain.yml

@@ -0,0 +1,14 @@
+---
+
+wordpress_image_version: latest
+
+connect_mail_protocol: "smtp"
+connect_mail_host: "smtp.office365.com"
+connect_mail_port: "587"
+connect_mail_user: "{{ connect_mail_user_vault }}"
+connect_mail_password: "{{ connect_mail_password_vault }}"
+connect_mail_properties_sender: "Info@egeld24.de"
+connect_mail_properties_sender_alias: "noreply"
+connect_mail_properties_smtp_auth: "true"
+connect_mail_properties_smtp_starttls_enable: "true"
+connect_mail_properties_smtp_starttls_required: "true"

host_vars/prodnso-mobene-cusqa-01/vault.yml

@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+34656337303930343532386532646463353864653937633637303733346462666333303034323037
+6633333162376661313838366334313034336162623164630a336132396361353431386135303439
+38383366616163363865366137316238666638383263326430653236383532303232636531323431
+3563623830303665610a356336363438373938373863663738633661616366323334323661346666
+61343632663635376264356263346430383236663363373331613639323065396533613635386531
+30646135333638343461386436663763393663313266363434623837373562636166393033396163
+65356633383732313034363965353162323230353263373537656539336364383935633436633334
+64633461336431353532323939303761653534313134326335363732623032306161653437353330
+38306561643033373033313963336164383235653639386261646134353237313639

host_vars/prodnso-mobene-nsodev-01.yml

@@ -0,0 +1,3 @@
+---
+
+wordpress_image_version: latest

host_vars/prodnso-postgres-01.yml

@@ -1,3 +1,4 @@
 ---
 
+hetzner_server_type: cpx21
 server_type: "master"

host_vars/prodnso-postgres-02.yml

@@ -1,3 +1,4 @@
 ---
 
+hetzner_server_type: cpx21
 server_type: "slave"

host_vars/prodnso-prometheus-01.yml

@@ -0,0 +1,3 @@
+---
+
+hetzner_server_type: cpx21

host_vars/prodwork01-postgres-01.yml

@@ -0,0 +1,4 @@
+---
+
+hetzner_server_type: cpx21
+server_type: "master"

host_vars/prodwork01-postgres-02.yml

@@ -0,0 +1,4 @@
+---
+
+hetzner_server_type: cpx21
+server_type: "slave"

kubernetes.yml

@@ -22,11 +22,30 @@
   roles:
     - { role: kubernetes/base }
     - { role: kubernetes/namespace }
-    - { role: kubernetes/cloud_controller_manager }
+    - role: kubernetes/cloud_controller_manager
+      when: kubernetes_with_ccm | default(True)
+      tags:
+        - ccm
     - { role: kubernetes/container_storage_interface }
-    - { role: kubernetes/prometheus }
+    - role: kubernetes/prometheus
-    - { role: kubernetes/cert_manager }
+      tags:
-    - { role: kubernetes/external_dns }
+        - prometheus
-    - { role: kubernetes/ingress_controller }
+      when: kubernetes_with_prometheus | default(True)
-    - { role: kubernetes/argocd }
+    - role: kubernetes/cert_manager
-    - { role: kubernetes/awx }
+      when: kubernetes_with_certmanager | default(True)
+    - role: kubernetes/external_dns
+      when: kubernetes_with_extdns | default(True)
+      tags:
+        - external-dns
+    - role: kubernetes/ingress_controller
+      when: kubernetes_with_ingress | default(True)
+    - role: kubernetes/argocd
+      when: kubernetes_with_argocd | default(True)
+      tags:
+        - argocd
+    - role: kubernetes/awx
+      when: kubernetes_with_awx | default(True)
+    - role: kubernetes/gitea
+      when: kubernetes_with_gitea | default(False)
+      tags:
+        - gitea

kubespray

@@ -1 +1 @@
-Subproject commit 92f25bf267ffd3393f6caffa588169d3a44a799c
+Subproject commit 00550ba832aa5d4f59bce03ead09d9e940e3a672

mobene.yml

@@ -0,0 +1,128 @@
+---
+
+# creates kubernetes namespace with secrets for usage with mobene
+# Parameters:
+#   secrets for mobene/namespaces read from group_vars
+
+- name: 'apply mobene setup to {{ host | default("kube_control_plane") }}'
+  hosts: '{{ host | default("kube_control_plane") }}'
+  serial: "{{ serial_number | default(10) }}"
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+      tags:
+        - always
+
+    - name: "Import autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      tags:
+        - always
+
+  roles:
+    - role: kubernetes/namespace
+      vars:
+        k8s_namespace: cus-mobene-nsodev
+        k8s_secrets:
+          - name: connect-secrets
+            data:
+              JWT_SECRET: "{{ mobene.nsodev.connect.secrets.JWT_SECRET | string | b64encode }}"
+              ADMIN_PASSWORD: "{{ mobene.nsodev.connect.secrets.ADMIN_PASSWORD | string | b64encode }}"
+              ELASTIC_USERNAME: "{{ mobene.nsodev.connect.secrets.ELASTIC_USERNAME | string | b64encode }}"
+              ELASTIC_PASSWORD: "{{ mobene.nsodev.connect.secrets.ELASTIC_PASSWORD | string | b64encode }}"
+              DATASOURCE_USERNAME: "{{ mobene.nsodev.connect.secrets.DATASOURCE_USERNAME | string | b64encode }}"
+              DATASOURCE_PASSWORD: "{{ mobene.nsodev.connect.secrets.DATASOURCE_PASSWORD | string | b64encode }}"
+              MAIL_USER: "{{ mobene.nsodev.connect.secrets.MAIL_USER | string | b64encode }}"
+              MAIL_PASSWORD: "{{ mobene.nsodev.connect.secrets.MAIL_PASSWORD | string | b64encode }}"
+              OIDC_CLIENT_SECRET: "{{ mobene.nsodev.connect.secrets.OIDC_CLIENT_SECRET | string | b64encode }}"
+          - name: iam-secrets
+            data:
+              JWT_SECRET: "{{ mobene.nsodev.iam.secrets.JWT_SECRET | string | b64encode }}"
+              KEYCLOAK_ADMIN_PASSWORD: "{{ mobene.nsodev.iam.secrets.KEYCLOAK_ADMIN_PASSWORD | string | b64encode }}"
+              KEYCLOAK_ADMIN_USERNAME: "{{ mobene.nsodev.iam.secrets.KEYCLOAK_ADMIN_USERNAME | string | b64encode }}"
+          - name: sepa-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.nsodev.sepaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.nsodev.sepaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: uba-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.nsodev.ubaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.nsodev.ubaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: wordpress-secrets
+            data:
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.nsodev.wordpress.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: elastic-client-cert
+            data:
+              ca.crt: "{{ mobene.nsodev.elastic.secrets.caCrt | string | b64encode }}"
+    - role: kubernetes/namespace
+      vars:
+        k8s_namespace: cus-mobene-cusqa
+        k8s_secrets:
+          - name: connect-secrets
+            data:
+              JWT_SECRET: "{{ mobene.cusqa.connect.secrets.JWT_SECRET | string | b64encode }}"
+              ADMIN_PASSWORD: "{{ mobene.cusqa.connect.secrets.ADMIN_PASSWORD | string | b64encode }}"
+              ELASTIC_USERNAME: "{{ mobene.cusqa.connect.secrets.ELASTIC_USERNAME | string | b64encode }}"
+              ELASTIC_PASSWORD: "{{ mobene.cusqa.connect.secrets.ELASTIC_PASSWORD | string | b64encode }}"
+              DATASOURCE_USERNAME: "{{ mobene.cusqa.connect.secrets.DATASOURCE_USERNAME | string | b64encode }}"
+              DATASOURCE_PASSWORD: "{{ mobene.cusqa.connect.secrets.DATASOURCE_PASSWORD | string | b64encode }}"
+              MAIL_USER: "{{ mobene.cusqa.connect.secrets.MAIL_USER | string | b64encode }}"
+              MAIL_PASSWORD: "{{ mobene.cusqa.connect.secrets.MAIL_USER | string | b64encode }}"
+              OIDC_CLIENT_SECRET: "{{ mobene.cusqa.connect.secrets.OIDC_CLIENT_SECRET | string | b64encode }}"
+          - name: iam-secrets
+            data:
+              JWT_SECRET: "{{ mobene.cusqa.iam.secrets.JWT_SECRET | string | b64encode }}"
+              KEYCLOAK_ADMIN_PASSWORD: "{{ mobene.cusqa.iam.secrets.KEYCLOAK_ADMIN_PASSWORD | string | b64encode }}"
+              KEYCLOAK_ADMIN_USERNAME: "{{ mobene.cusqa.iam.secrets.KEYCLOAK_ADMIN_USERNAME | string | b64encode }}"
+          - name: sepa-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.cusqa.sepaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusqa.sepaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: uba-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.cusqa.ubaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusqa.ubaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: wordpress-secrets
+            data:
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusqa.wordpress.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: elastic-client-cert
+            data:
+              ca.crt: "{{ mobene.cusqa.elastic.secrets.caCrt | string | b64encode }}"
+    - role: kubernetes/namespace
+      vars:
+        k8s_namespace: cus-mobene-cusprod
+        k8s_secrets:
+          - name: connect-secrets
+            data:
+              JWT_SECRET: "{{ mobene.cusprod.connect.secrets.JWT_SECRET | string | b64encode }}"
+              ADMIN_PASSWORD: "{{ mobene.cusprod.connect.secrets.ADMIN_PASSWORD | string | b64encode }}"
+              ELASTIC_USERNAME: "{{ mobene.cusprod.connect.secrets.ELASTIC_USERNAME | string | b64encode }}"
+              ELASTIC_PASSWORD: "{{ mobene.cusprod.connect.secrets.ELASTIC_PASSWORD | string | b64encode }}"
+              DATASOURCE_USERNAME: "{{ mobene.cusprod.connect.secrets.DATASOURCE_USERNAME | string | b64encode }}"
+              DATASOURCE_PASSWORD: "{{ mobene.cusprod.connect.secrets.DATASOURCE_PASSWORD | string | b64encode }}"
+              MAIL_USER: "{{ mobene.cusprod.connect.secrets.MAIL_USER | string | b64encode }}"
+              MAIL_PASSWORD: "{{ mobene.cusprod.connect.secrets.MAIL_USER | string | b64encode }}"
+              OIDC_CLIENT_SECRET: "{{ mobene.cusprod.connect.secrets.OIDC_CLIENT_SECRET | string | b64encode }}"
+          - name: iam-secrets
+            data:
+              JWT_SECRET: "{{ mobene.cusprod.iam.secrets.JWT_SECRET | string | b64encode }}"
+              KEYCLOAK_ADMIN_PASSWORD: "{{ mobene.cusprod.iam.secrets.KEYCLOAK_ADMIN_PASSWORD | string | b64encode }}"
+              KEYCLOAK_ADMIN_USERNAME: "{{ mobene.cusprod.iam.secrets.KEYCLOAK_ADMIN_USERNAME | string | b64encode }}"
+          - name: sepa-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.cusprod.sepaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusprod.sepaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: uba-exporter-secrets
+            data:
+              SMA_DOCUMENT_AUTH_TOKEN: "{{ mobene.cusprod.ubaExporter.secrets.SMA_DOCUMENT_AUTH_TOKEN | string | b64encode }}"
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusprod.ubaExporter.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: wordpress-secrets
+            data:
+              SMA_WORKFLOW_AUTH_TOKEN: "{{ mobene.cusprod.wordpress.secrets.SMA_WORKFLOW_AUTH_TOKEN | string | b64encode }}"
+          - name: elastic-client-cert
+            data:
+              ca.crt: "{{ mobene.cusprod.elastic.secrets.caCrt | string | b64encode }}"

patchday.yml

@@ -3,6 +3,42 @@
 ### tags:
 ###   check_elastic_cluster
 
+
+- hosts: prometheus
+  vars:
+    start: '{{ ansible_date_time.epoch }}'
+
+  tasks:
+    - name: "Set VAR for silence start and end"
+      set_fact:
+        silence_starts_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime(start) }}"
+        silence_ends_at: "{{ '%Y-%m-%d %H:%M:%S' | strftime( ( start | int ) + 3600 |int ) }}"
+
+    - name: "Set VAR - define prometheus silence object"
+      set_fact:
+        silence:
+          matchers:
+            - name: job
+              value: .+
+              isRegex: true
+          startsAt: '{{ silence_starts_at }}'
+          endsAt: '{{ silence_ends_at }}'
+          createdBy: patchday-automatism
+          comment: patchday
+          id:
+
+    - name: "Schedule silences for stage..."
+      uri:
+        url: "https://{{ stage }}-prometheus-01-alertmanager.smardigo.digital/api/v2/silences"
+        method: POST
+        status_code: [200]
+        headers:
+          Content-Type: application/json
+        body_format: json
+        body: '{{ silence | to_json }}'
+      ignore_errors: yes # noqa ignore-errors
+      # if failing, patchday continues
+
 - hosts: elastic
   serial: 1
   become: yes
@@ -102,38 +138,113 @@
         name: postgresql
         state: started
 
+    # wait_for cannot be used anymore due to enabled SSL encryption for postgres connections in DEV-382
     - name: "Smardigo Patchday: check if postgres is listing on net internal ip address"
-      ansible.builtin.wait_for:
+      become: no
-        delay: 15
+      community.postgresql.postgresql_ping:
-        timeout: 180
         port: 5432
-        host: '{{ stage_server_ip }}'
+        ssl_mode: require
+        login_host: '{{ stage_private_server_ip }}'
       register: check_postgres
+      ignore_errors: yes
 
-    - name: "Smardigo Patchday: restart postgres and check listing on net internal ip address again"
+    - name: "Smardigo Patchday: error-handling - ensure postgres started and check listing on net internal ip address"
       block:
 
-      - name: "Smardigo Patchday: stop service(s)"
+        - name: "Smardigo Patchday: error-handling - ensure service(s) started"
           ansible.builtin.systemd:
             name: postgresql
-          state: restarted
+            state: started
 
-      - name: "Smardigo Patchday: check if postgres is listing on net internal ip address"
+        - name: "Smardigo Patchday: error-handling - check if postgres is listing on net internal ip address"
+          become: no
+          community.postgresql.postgresql_ping:
+            port: 5432
+            ssl_mode: require
+            login_host: '{{ stage_private_server_ip }}'
+          register: check_postgres_again
+          retries: 5
+          failed_when: not check_postgres_again.is_available
+
+      rescue:
+        - name: "Smardigo Patchday: error-handling - send mail to DEVOPS-DL"
+          delegate_to: '{{ stage }}-mail-01'
+          community.general.mail:
+            host: localhost
+            port: 25
+            to: '{{ devops_email_address }}'
+            subject: "patchday( {{ lookup('pipe','date +%Y-%m-%d_%H:%M') }} ) problem report for {{ inventory_hostname }}"
+            body: |
+              Dear Sir or Madam,
+
+              I have to inform you that {{ inventory_hostname }} isn'n listening on {{ stage_private_server_ip }} anymore.
+
+              Plz check what happened/ fix it little padawan ;)
+
+              kind regards,
+
+              your automation-bofh
+
+      when:
+        - not check_postgres.is_available
+
+# due to bloody dependencies in SMA application startup, iam must be available during startup
+# => patching IAM service outsourced in separate part to make sure that is up and running
+- hosts: iam
+  serial: 10
+  become: yes
+  tasks:
+    - name: "Smardigo Patchday: update pkgs"
+      ansible.builtin.apt:
+        upgrade: yes
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+
+    - name: "Smardigo Patchday: find docker_compose.yml files"
+      ansible.builtin.find:
+        paths: '{{ service_base_path }}'
+        pattern: 'docker*.yml'
+        recurse: yes
+      register: docker_compose_services
+
+    - name: "Smardigo Patchday: shutdown services"
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: absent
+      loop: '{{ docker_compose_services.files }}'
+
+    - name: "Smardigo Patchday: rebooting <{{ inventory_hostname }}>"
+      ansible.builtin.reboot:
+        post_reboot_delay: 30
+        reboot_timeout: 300
+
+    - name: "Smardigo Patchday: wait_for host after reboot"
+      become: no
+      delegate_to: localhost
       ansible.builtin.wait_for:
         delay: 15
         timeout: 180
-          port: 5432
+        port: 22
         host: '{{ stage_server_ip }}'
-          register: check_postgres
+        search_regex: OpenSSH
-        failed_when: check_postgres_again.failed
 
-      when:
+    - name: "Smardigo Patchday: start services"
-        - check_postgres.failed
+      community.docker.docker_compose:
+        project_src: '{{ item.path | dirname }}'
+        state: present
+      loop: '{{ docker_compose_services.files }}'
 
-- hosts: all,!elastic,!postgres,!k8s_cluster
+- hosts: all,!elastic,!postgres,!k8s_cluster,!iam
   serial: 10
   become: yes
   tasks:
+    - name: "set VAR"
+      set_fact:
+        stage_server_ip: "{{ lookup('community.general.dig', inventory_hostname + '.' + domain ) }}"
+      when:
+        - "'blackbox' in group_names"
+
     - name: "Smardigo Patchday: update pkgs"
       ansible.builtin.apt:
         upgrade: yes
@@ -175,6 +286,58 @@
         state: present
       loop: '{{ docker_compose_services.files }}'
 
+    - name: "Ensure SMA-portal is up and running"
+      # there is a hard dependency within SMA-portal (VM: <<stage>>-management-01) during application start process
+      # to iam-instance (VM: <<stage>>-iam-01)
+      # grouped tasks within ansible block statement is just a work around until smardigo-app depending problem will be fixed
+      #
+      # ATTENTION: iam-server must be up and running => SMA-portal will be restarted and will finished successfully its application start process
+      block:
+        - name: "Check SMA-portal if reachable"
+          become: no
+          uri:
+            url: "https://{{ stage }}-management-01-connect.{{ domain }}:{{ admin_port_service }}/management/prometheus"
+            method: GET
+            status_code: [200]
+          register: sma_portal_avail
+          delay: 10
+          retries: 5
+          no_log: true
+          until: sma_portal_avail.status in [200]
+
+      rescue:
+        - name: "Check SMA-portal dependency << iam-instance >>is reachable"
+          become: no
+          uri:
+            url: "https://{{ stage }}-iam-01.{{ domain }}/api/v1/roles"
+            method: GET
+            status_code: [403]
+          register: iam_avail
+          delay: 10
+          retries: 10
+          no_log: true
+          until: iam.status in [403]
+          ignore_errors: yes # noqa ignore-errors
+          # patchday continues ion case of failed request towards iam service;
+          # iam service is hard dependency for SMA-portal-instance but not for
+          # patchday itself - it;s just a work around
+
+        - name: "Smardigo Patchday: SMA-portal not reachable - shutdown services"
+          community.docker.docker_compose:
+            project_src: '{{ item.path | dirname }}'
+            state: absent
+          loop: '{{ docker_compose_services.files }}'
+
+        - name: "Smardigo Patchday: SMA-portal not reachable - start services again"
+          community.docker.docker_compose:
+            project_src: '{{ item.path | dirname }}'
+            state: present
+          loop: '{{ docker_compose_services.files }}'
+
+      when:
+        - "'management' in inventory_hostname"
+
+
 - hosts: k8s_cluster
   serial: 1
   become: yes

pip-requirements

@@ -1,10 +1,11 @@
-ansible
+ansible==4.10.0
 ansible-builder
-ansible-core>=2.10
+ansible-core==2.11.11
-ansible-lint>=5.3.0
+ansible-lint==5.4.0
 dnspython
 hcloud>=1.16.0
 jmespath
 netaddr
 passlib>=1.7.4
 python-jose>=3.3.0
+bcrypt==4.0.0

provisioning.yml

@@ -2,7 +2,7 @@
 
 - name: 'apply setup to {{ host | default("all") }}'
   hosts: '{{ host | default("all") }}'
-  serial: "{{ serial_number | default(1) }}"
+  serial: "{{ serial_number | default(5) }}"
   gather_facts: no
   become: no
 
@@ -61,10 +61,11 @@
         hetzner_state: 'started'
       when:
         - "'hcloud' in group_names"
+
   tasks:
     - name: "Create server in DO-cloud via include_tasks"
       include_role:
-        name: sma_digitalocean
+        name: digitalocean
         tasks_from: _create_server
       vars:
         droplet:

remove-database.yml

@@ -84,18 +84,27 @@
     - role: connect_postgres
       when: "'connect' in group_names"
 
-    - role: pdns_admin_postgres
+    - role: gitea_postgres
-      when: "'pdns' in group_names"
+      when: "'gitea' in group_names"
-
-    - role: pdns_postgres
-      when: "'pdns' in group_names"
 
     - role: keycloak_postgres
       when: "'keycloak' in group_names"
 
+#    - role: pdns_admin_postgres
+#      when: "'pdns' in group_names"
+
+#    - role: pdns_postgres
+#      when: "'pdns' in group_names"
+
     - role: webdav_postgres
       when: "'webdav' in group_names"
 
+    - role: workflow_index_postgres
+      when: "'workflow_index' in group_names"
+
+    - role: workflow_proxy_postgres
+      when: "'workflow_proxy' in group_names"
+
     - role: connect_wordpress_maria
       when: "'connect_wordpress' in group_names"
 

remove-server.yml

@@ -73,7 +73,7 @@
 
     - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
       include_role:
-        name: sma_digitalocean
+        name: dns
         tasks_from: _remove_dns
       vars:
         record_to_remove: '{{ inventory_hostname }}'

remove-service.yml

@@ -63,7 +63,7 @@
   tasks:
     - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
       include_role:
-        name: sma_digitalocean
+        name: dns
         tasks_from: _remove_dns
       vars:
         record_to_remove: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-01-connect'
@@ -71,7 +71,7 @@
 
     - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
       include_role:
-        name: sma_digitalocean
+        name: dns
         tasks_from: _remove_dns
       vars:
         record_to_remove: '{{ stage }}-{{ tenant_id }}-{{ cluster_name }}-01-wordpress'

restore-database-backup.yml

@@ -61,7 +61,7 @@
   serial: "{{ serial_number | default(1) }}"
   remote_user: root
   vars:
-    postgres_backup_state: restore
+    database_backup_state: restore
     ansible_ssh_host: "{{ stage_server_domain }}"
 
   roles:

restore-remote-database-backup.yml

@@ -0,0 +1,265 @@
+---
+
+# restores remote database backup
+#   - postgres
+#     - executed on stage specific server: {{ stage }}-restore-postgres-01
+#     - restores a server from full-backup
+#   - mariadb
+#     - executed on stage specific server: {{ stage }}-restore-maria-01
+#     - restores a server from full-backup
+
+# Parameters:
+#   playbook inventory
+#     stage := the name of the stage (e.g. dev, int, qa, prod)
+#     database_engine := the database engine to restore a backup for (e.g. postgres, maria)
+#   smardigo message callback
+#     scope_id := (scope id of the management process)
+#     process_instance_id := (process instance id of the management process)
+#     smardigo_management_action := (smardigo management action anme of the management process)
+
+#############################################################
+#   Creating inventory dynamically for given parameters
+#############################################################
+
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  pre_tasks:
+    - name: "Check if ansible version is at least 2.10.x"
+      assert:
+        that:
+          - ansible_version.major >= 2
+          - ansible_version.minor >= 10
+        msg: "The ansible version has to be at least ({{ ansible_version.full }})"
+
+#     add virtual server to load stage specific variables as context
+    - name: "Add <{{ stage }}-virtual-host-to-read-groups-vars> to hosts"
+      add_host:
+        name: "{{ stage }}-virtual-host-to-read-groups-vars"
+        groups:
+        - "stage_{{ stage }}"
+      changed_when: False
+
+  tasks:
+    - name: "Add {{ database_engine }} servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-restore-{{ database_engine }}-01"
+        groups:
+        - "stage_{{ stage }}"
+        - 'restore'
+      changed_when: False
+    - name: "Add 'backup' servers to hosts if necessary"
+      add_host:
+        name: "{{ stage }}-backup-01"
+        groups:
+        - "stage_{{ stage }}"
+        - backup
+      changed_when: False
+
+#############################################################
+#   Create restore server(s)
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  remote_user: root
+
+  roles:
+    - role: hcloud
+      vars:
+        sma_digitalocean_ttl: 60 # set it to 60sec to reduce DNS caching problems with internal IT in case of debugging ansible problems ;)
+
+#############################################################
+#   Provisioning server(s) for created inventory
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  remote_user: root
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+
+  pre_tasks:
+    - name: "Import autodiscover pre-tasks"
+      import_tasks: tasks/autodiscover_pre_tasks.yml
+      become: false
+      tags:
+        - always
+
+  roles:
+    - role: common
+
+    - role: filebeat
+      when: filebeat_enabled | default(True)
+
+    - role: node_exporter
+      when: node_exporter_enabled | default(True)
+
+    - role: restore_{{ database_engine }}
+
+#############################################################
+#   add restore specific firewall rule
+#############################################################
+
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  vars:
+    hcloud_firewall_objects_backup:
+       -
+         name: "{{ stage }}-restore-ssh-access"
+         state: present
+         rules:
+         -
+            direction: in
+            protocol: tcp
+            port: '22'
+            source_ips:
+              - "{{ lookup('community.general.dig', groups['backup'][0] + '.' + domain ) }}/32"
+            destination_ips: []
+            description: null
+         apply_to:
+         -
+           type: label_selector
+           label_selector:
+             selector: 'service=restore'
+
+  tasks:
+    - name: "Add hcloud firewall rule(s)"
+      include_role:
+        name: hcloud
+        tasks_from: configure-firewall2
+      loop: "{{ hcloud_firewall_objects_backup }}"
+      loop_control:
+        loop_var: firewall_object
+
+#############################################################
+#   Syncing backups from backup server to restore server
+#############################################################
+
+- hosts: "backup"
+  serial: "{{ serial_number | default(5) }}"
+  gather_facts: false
+  vars:
+    backupserver_system_user: 'backuphamster'
+    ansible_ssh_host: "{{ stage_server_domain }}"
+  tasks:
+    # I could not get it up and running with <synchronize> module
+    # to sync data from remote server A to remote server B
+    - name: "Syncing remote backups"
+      become: yes
+      become_user: '{{ backupserver_system_user }}'
+      vars:
+        database_server_ip: "{{ groups['restore'][0] }}.{{ domain }}"
+      shell: '/home/{{ backupserver_system_user }}/push_backups_to_restore_server.sh {{ database_server_ip }} {{ stage }} {{ database_engine }}'
+
+#############################################################
+#   Restoring from backup
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+  tasks:
+    - name: "Triggering restore"
+      become: yes
+      shell: '/root/restore.sh {{ stage }}'
+
+    - name: "Check for test data on postgres"
+      block:
+
+      - name: "Querying postgres ..."
+        become: yes
+        become_user: postgres
+        community.postgresql.postgresql_query:
+          db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %(quote_val)s
+          named_args:
+            quote_val: 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_all_results | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'postgres'
+
+    - name: "Check for test data on mariadb"
+      block:
+
+      - name: "Querying mariadb ..."
+        become: yes
+        become_user: root
+        community.mysql.mysql_query:
+          login_unix_socket: /run/mysqld/mysqld.sock
+          login_db: dummytestdb
+          query: SELECT movie FROM movie_quotes WHERE quote = %s
+          positional_args:
+            - 'Shall we play'
+        register: query_output
+
+      - assert:
+          that:
+          - 'query_output.query_result | first | selectattr("movie","match","wargames") | length == 1'
+
+      when:
+        - database_engine == 'maria'
+
+
+#############################################################
+#   Deleting volumes for created inventory
+#############################################################
+- hosts: "restore"
+  serial: "{{ serial_number | default(5) }}"
+  gather_facts: false
+
+  tasks:
+    - name: "Delete volumes for <{{ inventory_hostname }}>"
+      include_role:
+        name: lvm_with_hetzner_volumes
+        tasks_from: _remove_hetzner_volumes
+
+#############################################################
+#   Deleting servers/domains for created inventory
+#############################################################
+
+- hosts: "restore"
+  serial: "{{ serial_number | default(5) }}"
+  gather_facts: false
+
+  tasks:
+    - name: "Delete server <{{ inventory_hostname }}>"
+      include_role:
+        name: hcloud
+        tasks_from: _set_server_state
+      vars:
+        - server_state: "absent"
+
+    - name: "Delete DNS entry <{{ inventory_hostname }}> for <{{ domain }}>"
+      include_role:
+        name: dns
+        tasks_from: _remove_dns
+      vars:
+        record_to_remove: '{{ inventory_hostname }}'
+
+#############################################################
+#   Sending smardigo management message to process
+#############################################################
+
+- hosts: "{{ stage }}-virtual-host-to-read-groups-vars"
+  serial: "{{ serial_number | default(1) }}"
+  gather_facts: false
+  connection: local
+  run_once: true
+  vars:
+    connect_jwt_username: "{{ management_admin_username }}"
+
+  tasks:
+    - name: "Sending smardigo management message to <{{ smardigo_management_url }}>"
+      include_tasks: tasks/smardigo_management_message.yml

roles/ansible-role-docker/tasks/main.yml

@@ -9,4 +9,4 @@
     name: "remove unused docker objects"
     minute: "0"
     hour: "1"
-    job: "docker system prune -af"
+    job: "docker system prune -af --filter label!=prune=disable"

roles/backup/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+system_user: backuphamster
+
+backup_lvm_hcloudvol_size: 10
+backup_lvm_hcloudvol_count: 1
+backup_lvm_hcloudvol_mountpath: '/home/{{ system_user }}/backups'

roles/backup/files/pull_remote_backups.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+#
+#
+
+# Fail fast and be aware of exit codes
+set -euo pipefail
+
+# Define some variables
+DATE=$(date +%F)
+DATE_TIME=$(date +%F_%H:%M)
+REMOTE_SYSTEM_USER=backupuser
+DATABASE_SERVER_IP=$1
+STAGE=$2
+DATABASE_ENGINE=$3
+DEST_DIR=${HOME}/backups/${STAGE}/${DATABASE_ENGINE}
+BACKUP_STATUS_FILE=${DEST_DIR}/${DATE}/backup_finished_${DATE}_*
+METRICS_FILE=${HOME}/backup_status_${DATABASE_ENGINE}.prom
+LOG_FILE=${DEST_DIR}/backup_${DATE_TIME}.log
+
+# Redirect stderr to stdout and save everything to log file
+exec > ${LOG_FILE} 2>&1
+
+# Log backup sync start time
+echo "----- Start backup Sync - ${DATE_TIME} -----"
+
+# Create backup directory ${DEST_DIR} if not exist
+mkdir -p ${DEST_DIR}
+
+# Remove files oder than 48h in ${DEST_DIR}
+find $DEST_DIR -type d -mtime +1 -print0 | xargs -I OLD_DIR -0 rm -rf "OLD_DIR"
+[ "$?" != "0" ] && exit 1
+
+# Start rsync job from ${DATABASE_SERVER_IP} to ${DEST_DIR}/
+rsync -av --remove-source-files -e "ssh -o StrictHostKeyChecking=no" ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:/backups/${DATABASE_ENGINE}/ ${DEST_DIR}/
+[ "$?" -eq "0" ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1"
+
+# Check existence of current ${BACKUP_STATUS_FILE}, which is created by AWX, in case of succesful database backup only.
+[ -f ${BACKUP_STATUS_FILE} ] && NIGHTLY_BACKUP_SUCCESSFUL="0" || NIGHTLY_BACKUP_SUCCESSFUL="1"
+
+# Add backup status to Prometheus metrics file
+cat <<EOF > $METRICS_FILE
+# HELP nightly_backup_successful_${DATABASE_ENGINE}
+# TYPE nightly_backup_successful_${DATABASE_ENGINE} gauge
+nightly_backup_successful_${DATABASE_ENGINE}{stage="$STAGE"} $NIGHTLY_BACKUP_SUCCESSFUL
+EOF
+
+# Log backup sync end time
+echo "----- End backup Sync - ${DATE_TIME} -----"

roles/backup/files/push_backups_to_restore_server.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+#
+#
+
+REMOTE_SYSTEM_USER=backupuser
+DATABASE_SERVER_IP=$1
+STAGE=$2
+DATABASE_ENGINE=$3
+
+# currently it defaults to todays date
+DATE=$(date +%F)
+
+LOCAL_BACKUP_DIR="${HOME}/backups/${STAGE}/${DATABASE_ENGINE}"
+BACKUP_FILE_FOR_TRANSFER=$(find "${LOCAL_BACKUP_DIR}/${DATE}/" -name *.gz.gpg | tail -n 1)
+
+REMOTE_BACKUP_DIR="/home/${REMOTE_SYSTEM_USER}/backups/${STAGE}/${DATABASE_ENGINE}"
+DEST_DIR="${REMOTE_BACKUP_DIR}/${DATE}/"
+
+if [ ! -f $BACKUP_FILE_FOR_TRANSFER ]; then
+   echo "BACKUP_FILE_FOR_TRANSFER not found. EXIT" && exit 1
+fi
+
+# avoid "REMOTE HOST IDENTIFICATION HAS CHANGED" - errors due to dynamic created server on restore process
+ssh-keygen -f "/home/backuphamster/.ssh/known_hosts" -R ${DATABASE_SERVER_IP}
+
+SSH_OPTIONS='-o StrictHostKeyChecking=no'
+
+# needed due to unknown rsync option --mkpath in rsync version 3.1.3
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "mkdir -p ${DEST_DIR}"
+
+rsync -v -e "ssh ${SSH_OPTIONS}" $BACKUP_FILE_FOR_TRANSFER ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP}:${DEST_DIR}
+
+BKP_FILE_TRANSFERRED=$(echo $BACKUP_FILE_FOR_TRANSFER | awk -F / '{ print $NF}')
+
+ssh ${SSH_OPTIONS} ${REMOTE_SYSTEM_USER}@${DATABASE_SERVER_IP} "test -f ${DEST_DIR}${BKP_FILE_TRANSFERRED}"
+

roles/backup/tasks/main.yml

@@ -0,0 +1,96 @@
+---
+
+- name: "Backup storage server | create system user"
+  become: yes
+  ansible.builtin.user:
+    name: '{{ system_user }}'
+    comment: "user for backup"
+    shell: /bin/bash
+  register: create_user
+
+- name: "Create .ssh dir and backups dir"
+  become: yes
+  file:
+    path: '/home/{{ system_user }}/{{ item.name }}/'
+    mode: '{{ item.mode }}'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    state: directory
+  loop:
+    - name: '.ssh'
+      mode: '0700'
+    - name: 'backups'
+      mode: '0775'
+
+- name: "Create/Resize LVM for datadir"
+  include_role:
+    name: lvm_with_hetzner_volumes
+  vars:
+    lvm_with_hetzner_volumes__volprefix: backup_datadir
+    lvm_with_hetzner_volumes__volsize: "{{ backup_lvm_hcloudvol_size }}"
+    lvm_with_hetzner_volumes__volcount: "{{ backup_lvm_hcloudvol_count }}"
+    lvm_with_hetzner_volumes__mountpath: "{{ backup_lvm_hcloudvol_mountpath }}"
+
+- name: "Providing SSH priv.key"
+  no_log: true
+  become: yes
+  copy:
+    dest: '/home/{{ system_user }}/.ssh/id_rsa'
+    mode: '0400'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+    content: '{{ backup_user_ssh_privkey_vault }}'
+
+- name: "Providing rsync script"
+  become: yes
+  copy:
+    src: '{{ item }}'
+    dest: '/home/{{ system_user }}/{{ item }}'
+    mode: '0755'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+  with_items:
+    - pull_remote_backups.sh
+    - push_backups_to_restore_server.sh
+
+- name: Touch metrics.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/metrics.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Touch backup_status_maria.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/backup_status_maria.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Touch backup_status_postgres.prom if not exists
+  file:
+    path: "/home/{{ system_user }}/backup_status_postgres.prom"
+    state: touch
+    mode: '0744'
+    owner: '{{ system_user }}'
+    group: '{{ system_user }}'
+
+- name: Create symbolic link for node_exporter text metrics
+  file:
+    src: "/home/{{ system_user }}/metrics.prom"
+    dest: "/var/lib/prometheus/node-exporter/offsite-metrics.prom"
+    state: link
+
+- name: Create symbolic link for node_exporter text metrics backup_status_maria
+  file:
+    src: "/home/{{ system_user }}/backup_status_maria.prom"
+    dest: "/var/lib/prometheus/node-exporter/backup_status_maria.prom"
+    state: link
+
+- name: Create symbolic link for node_exporter text metrics backup_status_postgres
+  file:
+    src: "/home/{{ system_user }}/backup_status_postgres.prom"
+    dest: "/var/lib/prometheus/node-exporter/backup_status_postgres.prom"
+    state: link

roles/common/tasks/main.yml

@@ -23,6 +23,8 @@
       {% for host in shared_service_hosts %}
       {{ host.ip }} {{ host.name }}
       {% endfor %}
+  when:
+    - "'hcloud' in group_names"
   tags:
     - update_etc_hosts
 
@@ -59,7 +61,7 @@
 - name: "Remove outdated users"
   user: name={{ item }} state=absent remove=yes
   with_items: "{{ current_users.stdout_lines }}"
-  when: not ((item in default_plattform_users) or (item in smardigo_plattform_users))
+  when: not ((item in default_users) or (item in smardigo_plattform_users))
   tags:
     - users
 
@@ -97,24 +99,13 @@
   tags:
     - users
 
-- name: "Create stuff for backups on database servers"
+- name: "Update available package list"
-  block:
+  apt:
-  - name: "Create system user for remote_backup"
+    update_cache: yes
-    become: yes
-    ansible.builtin.user:
-      name: '{{ backupuser_username }}'
-      comment: "user for backup"
-      shell: /bin/bash
-
-  - name: "Add SSH pub key to auth_keys"
-    authorized_key:
-      user: '{{ backupuser_username }}'
-      key: '{{ backupuser_ssh_pubkey }}'
-  when:
-    - inventory_hostname in groups['postgres'] or
-      inventory_hostname in groups['maria']
   tags:
-    - users
+    - install
+    - upgrade
+  when: ansible_distribution == "Ubuntu"
 
 - name: "Ensure docker configuration directory exists"
   file:
@@ -245,6 +236,8 @@
 - name: "Create Docker network"
   community.docker.docker_network:
     name: "{{ item }}"
+    labels:
+      prune: disable
   when: docker_enabled
   loop:
     - front-tier
@@ -279,3 +272,13 @@
     state: present
   tags:
     - config
+
+- name: "configure ssh_hardening"
+  include_role:
+    # include role from collection called 'devsec'
+    name: devsec.hardening.ssh_hardening
+    apply:
+      tags:
+        - ssh_hardening
+  tags:
+    - ssh_hardening

roles/confirm_postgres/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+confirm_postgres_database: '{{ stage }}_{{ tenant_id }}_{{ cluster_name }}_confirm'
+confirm_postgres_password: 'confirm-postgres-admin'
+
+postgres_acls:
+  - name: "{{ confirm_postgres_database }}"
+    password: "{{ confirm_postgres_password }}"
+    trusted_cidr_entry: "{{ shared_service_network }}"

roles/confirm_postgres/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+
+### tags:
+
+- name: "Updating <confirm> database on {{ inventory_hostname }}"
+  include_role:
+    name: postgres
+    tasks_from: _update_database_state
+  when:
+  - database_backup_state is not defined
+
+- name: "Creating/Restoring <confirm> database backup on {{ inventory_hostname }}"
+  include_role:
+    name: postgres
+    tasks_from: _create_database_backup.yml
+  when:
+  - database_backup_state is defined
+  - database_backup_state in ['dump', 'restore']

roles/connect/defaults/main.yml

@@ -2,10 +2,6 @@
 
 connect_image_name: "{{ shared_service_harbor_hostname }}/smardigo/connect-whitelabel-app"
 
-# TODO inject by management portal
-connect_admin_username: "connect-admin"
-connect_admin_password: "connect-admin"
-
 connect_mail_host: "{{ shared_service_mail_hostname }}"
 connect_mail_properties_base_url: "{{ http_s }}://{{ connect_base_url }}"
 connect_mail_properties_base_url_extern: "{{ http_s }}://{{ connect_base_url }}"

roles/connect/tasks/main.yml

@@ -3,13 +3,10 @@
 ### tags:
 ###   update_certs
 ###   update_deployment
-###   update_connections
-###   update_configuration
 
 - name: "Setup DNS configuration for <{{ connect_id }}> to <{{ stage_server_ip }}>"
   include_role:
-    name: sma_digitalocean
+    name: dns
-    tasks_from: domain
   vars:
     record_data: "{{ stage_server_ip }}"
     record_name: "{{ connect_id }}"
@@ -66,6 +63,14 @@
   tags:
     - update_certs
 
+- name: "Restart {{ connect_id }}"
+  community.docker.docker_compose:
+    project_src: '{{ service_base_path }}/{{ connect_id }}'
+    restarted: yes
+    build: no
+  tags:
+    - update_certs
+
 - name: "Update {{ connect_id }}"
   community.docker.docker_compose:
     project_src: '{{ service_base_path }}/{{ connect_id }}'
@@ -73,10 +78,3 @@
     pull: yes
   tags:
     - update_deployment
-
-- name: "Configure connect connections"
-  include_tasks: connections.yml
-  when:
-    smardigo_auth_token_value is defined
-  tags:
-    - always

roles/connect/vars/main.yml

@@ -20,8 +20,8 @@ connect_labels: [
 connect_environment: [
   "TENANT_ID: \"{{ connect_client_id }}\"",
 
-  "ADMIN_LOGIN: \"{{ connect_admin_username }}\"",
+  "ADMIN_LOGIN: \"{{ connect_client_admin_username }}\"",
-  "ADMIN_PASSWORD: \"{{ connect_admin_password }}\"",
+  "ADMIN_PASSWORD: \"{{ connect_client_admin_password }}\"",
   "SMA_JWT_ENABLED: \"{{ connect_jwt_enabled | default('false') }}\"",
   "SMA_JWT_SECRET: \"{{ connect_jwt_secret | default('') }}\"",
   "SMA_CSRF_TOKEN_NAME: \"{{ connect_csrf_token_name | default('') }}\"",
@@ -29,7 +29,7 @@ connect_environment: [
   "SPRING_PROFILES_INCLUDE: \"{{ spring_profiles_include | default('swagger') }}\"",
   "RIBBON_DISPLAY_ON_ACTIVE_PROFILES: \"{{ ribbon_display_on_active_profiles | default('dev') }}\"",
 
-  "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_host }}:{{ service_port_postgres }}/{{ connect_postgres_database }}\"",
+  "DATASOURCE_URL: \"jdbc:postgresql://{{ connect_postgres_host }}:{{ service_port_postgres }}/{{ connect_postgres_database }}?sslmode=require\"",
   "DATASOURCE_USERNAME: \"{{ connect_postgres_username }}\"",
   "DATASOURCE_PASSWORD: \"{{ connect_postgres_password }}\"",
   "FILE_WHITELIST_URL: \"{{ connect_whitelist_url | default('') }}\"",
@@ -44,6 +44,9 @@ connect_environment: [
   "MAIL_PROPERTIES_BASE_URL_EXTERN: \"{{ connect_mail_properties_base_url_extern }}\"",
   "MAIL_PROPERTIES_SENDER: \"{{ connect_mail_properties_sender | default('noreply-connect@netgo.de') }}\"",
   "MAIL_PROPERTIES_SENDER_ALIAS: \"{{ connect_mail_properties_sender_alias | default('noreply-connect') }}\"",
+  "MAIL_PROPERTIES_SMTP_AUTH: \"{{ connect_mail_properties_smtp_auth | default('false') }}\"",
+  "MAIL_PROPERTIES_SMTP_STARTTLS_ENABLE: \"{{ connect_mail_properties_smtp_starttls_enable | default('false') }}\"",
+  "MAIL_PROPERTIES_SMTP_STARTTLS_REQUIRED: \"{{ connect_mail_properties_smtp_starttls_required | default('false') }}\"",
 
   "AUTH_MODULE: \"{{ connect_auth_module | default('preauth') }}\"",
   "OIDC_CLIENT_ID: \"{{ connect_oidc_client_id | default('oidc_config_not_found') }}\"",
@@ -99,6 +102,10 @@ connect_environment: [
   "OPENTRACING_JAEGER_LOG_SPANS: \"{{ connect_opentracing_jaeger_log_spans | default(false) }}\"",
   "OPENTRACING_JAEGER_SERVICE_NAME: \"{{ connect_opentracing_jaeger_service_name | default(connect_id) }}\"",
   "OPENTRACING_JAEGER_HTTP_SENDER_URL: \"{{ connect_opentracing_jaeger_http_sender_url | default() }}\"",
+
+  "CONFIG_DELETE_SCOPE_ENABLED: \"{{ connect_config_delete_scope_enabled | default(false) }}\"",
+  "CONFIG_LOCAL_IMPORT_ENABLED: \"{{ connect_config_local_import_enabled | default(false) }}\"",
+  "SMA_WORKFLOW_HEATMAP_ENABLED: \"{{ connect_workflow_heatmap_enabled | default(false) }}\"",
 ]
 
 connect_docker: {
@@ -116,7 +123,7 @@ connect_docker: {
     {
       name: "{{ connect_id }}",
       image_name: "{{ connect_image_name }}",
-      image_version: "{{ connect_image_version }}",
+      image_version: "{{ connect_version }}",
       labels: "{{ connect_labels + ( connect_labels_additional | default([])) }}",
       restart: "{{ connect_service_restart | default('always') }}",
       environment: "{{ connect_environment + ( connect_environment_additional | default([])) }}",

roles/connect_compact/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+
+connect_id: "{{ inventory_hostname }}-connect"
+connect_admin_username: "connect-admin"
+connect_admin_password: "{{ connect_admin_password_vault }}"
+connect_postgres_username: "connect-postgres-username"
+connect_postgres_password: "{{ connect_postgres_password_vault }}"
+connect_image_name: "{{ shared_service_harbor_hostname }}/smardigo/connect-whitelabel-app"
+
+keycloak_id: "{{ inventory_hostname }}-keycloak"
+keycloak_admin_username: "keycloak-admin"
+keycloak_admin_password: "{{ keycloak_admin_password_vault }}"
+keycloak_postgres_username: "keycloak_postgres"
+keycloak_postgres_password: "{{ keycloak_postgres_password_vault }}"
+keycloak_image_name: "{{ shared_service_harbor_hostname }}/smardigo/keycloak"
+
+elasticsearch_id: "{{ inventory_hostname }}-elastic"
+elasticsearch_username: "elastic"
+elasticsearch_password: "{{ elasticsearch_password_vault }}"

roles/connect_compact/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+
+- name: "Setup DNS configuration for <{{ connect_id }}> to <{{ stage_server_ip }}>"
+  include_role:
+    name: dns
+  vars:
+    record_data: "{{ stage_server_ip }}"
+    record_name: "{{ connect_id }}"
+
+- name: "Setup DNS configuration for <{{ connect_external_domain }}> to <{{ stage_server_ip }}>"
+  include_role:
+    name: dns
+  vars:
+    record_data: "{{ stage_server_ip }}"
+    record_name: "{{ connect_external_domain }}"
+  when: connect_external_domain is defined
+
+- name: "Check if {{ connect_id }}/docker-compose.yml exists"
+  stat:
+    path: '{{ service_base_path }}/{{ connect_id }}/docker-compose.yml'
+  register: check_docker_compose_file_connect
+
+- name: "Stop {{ connect_id }}"
+  community.docker.docker_compose:
+    project_src: '{{ service_base_path }}/{{ connect_id }}'
+    state: absent
+  when: check_docker_compose_file_connect.stat.exists
+
+- name: "Deploy docker templates for {{ connect_id }}"
+  include_role:
+    name: sma_deploy
+    tasks_from: templates
+  vars:
+    current_config: "connect-compact"
+    current_base_path: "{{ service_base_path }}"
+    current_destination: "{{ connect_id }}"
+    current_owner: "{{ docker_owner }}"
+    current_group: "{{ docker_group }}"
+
+- name: "Restart {{ connect_id }}"
+  community.docker.docker_compose:
+    project_src: '{{ service_base_path }}/{{ connect_id }}'
+    restarted: yes
+    build: no