DEV-548: testcluster

feature/DEV-655
Görz, Friedrich 3 years ago committed by Ketelsen, Sven
parent bad109ad83
commit 01049bf031

@ -97,11 +97,11 @@ hcloud_firewall_objects:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=connect' selector: 'stage={{ stage }},service=connect'
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=keycloak' selector: 'stage={{ stage }},service=keycloak'
hcloud_firewall_objects_awx: hcloud_firewall_objects_awx:
- -
@ -119,7 +119,7 @@ hcloud_firewall_objects_awx:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'stage={{ stage }}' selector: 'stage={{ stage }},service=awx'
hcloud_firewall_objects_backup: hcloud_firewall_objects_backup:
- -
@ -138,7 +138,7 @@ hcloud_firewall_objects_backup:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=backup' selector: 'stage={{ stage }},service=backup'
hcloud_firewall_objects_gitea: hcloud_firewall_objects_gitea:
- -
@ -170,7 +170,7 @@ hcloud_firewall_objects_gitea:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=gitea' selector: 'stage={{ stage }},service=gitea'
hcloud_firewall_objects_keycloak: hcloud_firewall_objects_keycloak:
- -
@ -202,7 +202,7 @@ hcloud_firewall_objects_keycloak:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=keycloak' selector: 'stage={{ stage }},service=keycloak'
hcloud_firewall_objects_kibana: hcloud_firewall_objects_kibana:
- -
@ -234,7 +234,7 @@ hcloud_firewall_objects_kibana:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=kibana' selector: 'stage={{ stage }},service=kibana'
hcloud_firewall_objects_management: hcloud_firewall_objects_management:
- -
@ -266,4 +266,4 @@ hcloud_firewall_objects_management:
- -
type: label_selector type: label_selector
label_selector: label_selector:
selector: 'service=connect,tenant=management' selector: 'stage={{ stage }},service=connect,tenant=management'

@ -30,11 +30,15 @@
- role: kubernetes/prometheus - role: kubernetes/prometheus
tags: tags:
- prometheus - prometheus
- { role: kubernetes/cert_manager } when: kubernetes_with_prometheus | default(True)
- role: kubernetes/cert_manager
when: kubernetes_with_certmanager | default(True)
- role: kubernetes/external_dns - role: kubernetes/external_dns
when: kubernetes_with_extdns | default(True)
tags: tags:
- external-dns - external-dns
- { role: kubernetes/ingress_controller } - role: kubernetes/ingress_controller
when: kubernetes_with_ingress | default(True)
- role: kubernetes/argocd - role: kubernetes/argocd
when: kubernetes_with_argocd | default(True) when: kubernetes_with_argocd | default(True)
tags: tags:

@ -2,7 +2,7 @@
- name: "Get all existing firewalls" - name: "Get all existing firewalls"
uri: uri:
method: GET method: GET
url: "https://api.hetzner.cloud/v1/firewalls" url: "https://api.hetzner.cloud/v1/firewalls?per_page=1000"
body_format: json body_format: json
headers: headers:
accept: application/json accept: application/json

@ -28,7 +28,7 @@ argo_realm_users: [
"requiredActions": [] "requiredActions": []
} }
] ]
argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" argocd_server_admin_password: "{{ argocd_server_admin_password_vault | default( lookup('community.general.random_string', length=20) ) }}"
# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd # https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
k8s_argocd_helm__release_values: k8s_argocd_helm__release_values:

@ -1,215 +1,208 @@
--- ---
- name: "Login with keycloak-admin"
include_role:
name: keycloak
tasks_from: _authenticate
args:
apply:
tags:
- argo-cd
when:
- k8s_argocd_with_keycloak
tags:
- argo-cd
- name: "Setup keycloak-realm for argocd" - name: "Do some stuff with keycloak as OIDC provider"
include_role: block:
name: keycloak - name: "Login with keycloak-admin"
tasks_from: _configure_realm include_role:
vars: name: keycloak
current_realm_name: '{{ argo_realm_name }}' tasks_from: _authenticate
current_realm_display_name: '{{ argo_realm_display_name }}' args:
create_client: False apply:
current_realm_password_policy: '' tags:
when: - argo-cd
- k8s_argocd_with_keycloak when:
- inventory_hostname == groups['kube_control_plane'][0] tags:
args: - argo-cd
apply:
tags:
- argo-cd
tags:
- argo-cd
- name: "Create a Keycloak group, authentication with credentials" - name: "Setup keycloak-realm for argocd"
include_role: include_role:
name: keycloak name: keycloak
tasks_from: _create_realm_groups tasks_from: _configure_realm
vars: vars:
current_realm_name: '{{ argo_realm_name }}' current_realm_name: '{{ argo_realm_name }}'
current_realm_display_name: '{{ argo_realm_display_name }}' current_realm_display_name: '{{ argo_realm_display_name }}'
current_realm_groups: create_client: False
- name: "{{ argo_realm_group }}" current_realm_password_policy: ''
when: when:
- k8s_argocd_with_keycloak - inventory_hostname == groups['kube_control_plane'][0]
- inventory_hostname == groups['kube_control_plane'][0] args:
args: apply:
apply: tags:
tags: - argo-cd
- argo-cd tags:
tags: - argo-cd
- argo-cd
- name: "Create keycloak user(s)" - name: "Create a Keycloak group, authentication with credentials"
include_role: include_role:
name: keycloak name: keycloak
tasks_from: _create_realm_users tasks_from: _create_realm_groups
vars: vars:
current_realm_name: '{{ argo_realm_name }}' current_realm_name: '{{ argo_realm_name }}'
current_realm_users: '{{ argo_realm_users }}' current_realm_display_name: '{{ argo_realm_display_name }}'
when: current_realm_groups:
- k8s_argocd_with_keycloak - name: "{{ argo_realm_group }}"
- inventory_hostname == groups['kube_control_plane'][0] when:
args: - inventory_hostname == groups['kube_control_plane'][0]
apply: args:
tags: apply:
- argo-cd tags:
tags: - argo-cd
- argo-cd tags:
- argo-cd
- name: "ADD user group mapping" - name: "Create keycloak user(s)"
include_role: include_role:
name: keycloak name: keycloak
tasks_from: _configure_user_groupmembership_crud tasks_from: _create_realm_users
vars: vars:
username: '{{ argocd_admin_username }}' current_realm_name: '{{ argo_realm_name }}'
destination_group: '{{ argo_realm_group }}' current_realm_users: '{{ argo_realm_users }}'
realm_name: '{{ argo_realm_name }}' when:
bearer_token: '{{ access_token }}' - inventory_hostname == groups['kube_control_plane'][0]
when: args:
- k8s_argocd_with_keycloak apply:
- inventory_hostname == groups['kube_control_plane'][0] tags:
args: - argo-cd
apply: tags:
tags: - argo-cd
- argo-cd
tags:
- argo-cd
- name: "Create keycloak clientscope" - name: "ADD user group mapping"
delegate_to: localhost include_role:
become: False name: keycloak
community.general.keycloak_clientscope: tasks_from: _configure_user_groupmembership_crud
auth_client_id: admin-cli vars:
auth_keycloak_url: "{{ keycloak_server_url }}/auth" username: '{{ argocd_admin_username }}'
auth_realm: 'master' destination_group: '{{ argo_realm_group }}'
auth_username: "{{ keycloak_admin_username }}" realm_name: '{{ argo_realm_name }}'
auth_password: "{{ keycloak_admin_password }}" bearer_token: '{{ access_token }}'
name: '{{ argo_keycloak_clientscope_name }}' when:
realm: '{{ argo_realm_name }}' - inventory_hostname == groups['kube_control_plane'][0]
protocol: '{{ argo_keycloak_clientscope_protocol }}' args:
protocol_mappers: apply:
- config: tags:
access.token.claim: True - argo-cd
claim.name: '{{ argo_keycloak_clientscope_name }}' tags:
full.path: False # set it to true and you will be DAMNED => groupname for argo k8s configmap argocd-rbac-cm will be "/{{ group_name }}" !!!! instead of "{{ group_name }}" - argo-cd
id.token.claim: True
userinfo.token.claim: True
name: '{{ argo_keycloak_clientscope_name }}'
protocol: openid-connect
protocolMapper: oidc-group-membership-mapper
when:
- k8s_argocd_with_keycloak
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
# using template from exported keycloak client object - name: "Create keycloak clientscope"
# due to needed params but missing in community.general.keycloak_client delegate_to: localhost
# e.g. defaultClientScopes become: False
- name: "Create json object as VAR from template" community.general.keycloak_clientscope:
set_fact: auth_client_id: admin-cli
keycloak_realm_create_client: "{{ lookup('template','keycloak-realm-create-client-argocd.json.j2') }}" auth_keycloak_url: "{{ keycloak_server_url }}/auth"
vars: auth_realm: 'master'
client_redirect_uri: '{{ argo_client_redirect_uris }}' auth_username: "{{ keycloak_admin_username }}"
client_web_origins: '{{ argo_client_web_origins }}' auth_password: "{{ keycloak_admin_password }}"
client_id: '{{ argo_client_id }}' name: '{{ argo_keycloak_clientscope_name }}'
realm_name: '{{ argo_realm_name }}' realm: '{{ argo_realm_name }}'
client_root_url: '{{ argo_client_root_url }}' protocol: '{{ argo_keycloak_clientscope_protocol }}'
client_admin_url: '{{ argo_client_admin_url }}' protocol_mappers:
client_base_url: '{{ argo_client_base_url }}' - config:
keycloak_clientscope_name: '{{ argo_keycloak_clientscope_name }}' access.token.claim: True
keycloak_clientscope_protocol: '{{ argo_keycloak_clientscope_protocol }}' claim.name: '{{ argo_keycloak_clientscope_name }}'
keycloak_client_secret: '{{ argo_keycloak_client_secret }}' full.path: False # set it to true and you will be DAMNED => groupname for argo k8s configmap argocd-rbac-cm will be "/{{ group_name }}" !!!! instead of "{{ group_name }}"
when: id.token.claim: True
- k8s_argocd_with_keycloak userinfo.token.claim: True
tags: name: '{{ argo_keycloak_clientscope_name }}'
- argo-cd protocol: openid-connect
protocolMapper: oidc-group-membership-mapper
when:
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
# throw needed VARs against keycloak API # using template from exported keycloak client object
# to CRUD # due to needed params but missing in community.general.keycloak_client
- name: "Create client" # e.g. defaultClientScopes
include_role: - name: "Create json object as VAR from template"
name: keycloak set_fact:
tasks_from: _configure_client_crud keycloak_realm_create_client: "{{ lookup('template','keycloak-realm-create-client-argocd.json.j2') }}"
vars: vars:
client_id: '{{ argo_client_id }}' client_redirect_uri: '{{ argo_client_redirect_uris }}'
realm_name: '{{ argo_realm_name }}' client_web_origins: '{{ argo_client_web_origins }}'
keycloak_client_object: '{{ keycloak_realm_create_client }}' client_id: '{{ argo_client_id }}'
bearer_token: '{{ access_token }}' realm_name: '{{ argo_realm_name }}'
when: client_root_url: '{{ argo_client_root_url }}'
- k8s_argocd_with_keycloak client_admin_url: '{{ argo_client_admin_url }}'
- inventory_hostname == groups['kube_control_plane'][0] client_base_url: '{{ argo_client_base_url }}'
args: keycloak_clientscope_name: '{{ argo_keycloak_clientscope_name }}'
apply: keycloak_clientscope_protocol: '{{ argo_keycloak_clientscope_protocol }}'
tags: keycloak_client_secret: '{{ argo_keycloak_client_secret }}'
- argo-cd tags:
tags: - argo-cd
- argo-cd
- name: "GET available clients from <<{{ argo_realm_name }}>>-realm" # throw needed VARs against keycloak API
delegate_to: localhost # to CRUD
become: False - name: "Create client"
uri: include_role:
url: "{{ keycloak_server_url }}/auth/admin/realms/{{ argo_realm_name }}/clients" name: keycloak
method: GET tasks_from: _configure_client_crud
headers: vars:
Content-Type: "application/json" client_id: '{{ argo_client_id }}'
Authorization: "Bearer {{ access_token }}" realm_name: '{{ argo_realm_name }}'
status_code: [200] keycloak_client_object: '{{ keycloak_realm_create_client }}'
register: argo_realm_clients bearer_token: '{{ access_token }}'
when: when:
- k8s_argocd_with_keycloak - inventory_hostname == groups['kube_control_plane'][0]
- inventory_hostname == groups['kube_control_plane'][0] args:
tags: apply:
- argo-cd tags:
- argo-cd
tags:
- argo-cd
# available clients: get needed ID - name: "GET available clients from <<{{ argo_realm_name }}>>-realm"
- name: "Get ID of client by paring argo_realm_clients object" delegate_to: localhost
set_fact: become: False
id_of_client: '{{ ( argo_realm_clients.json | selectattr("clientId","equalto",argo_client_id ) | first ).id }}' uri:
when: url: "{{ keycloak_server_url }}/auth/admin/realms/{{ argo_realm_name }}/clients"
- k8s_argocd_with_keycloak method: GET
- inventory_hostname == groups['kube_control_plane'][0] headers:
tags: Content-Type: "application/json"
- argo-cd Authorization: "Bearer {{ access_token }}"
status_code: [200]
register: argo_realm_clients
when:
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
- name: "GET client-secret for client <<{{ argo_client_id }}>> in realm <<{{ argo_realm_name }}>>" # available clients: get needed ID
delegate_to: localhost - name: "Get ID of client by paring argo_realm_clients object"
become: False set_fact:
uri: id_of_client: '{{ ( argo_realm_clients.json | selectattr("clientId","equalto",argo_client_id ) | first ).id }}'
url: "{{ keycloak_server_url }}/auth/admin/realms/{{ argo_realm_name }}/clients/{{ id_of_client }}/client-secret" when:
method: GET - inventory_hostname == groups['kube_control_plane'][0]
headers: tags:
Content-Type: "application/json" - argo-cd
Authorization: "Bearer {{ access_token }}"
status_code: [200]
register: client_secret
when:
- k8s_argocd_with_keycloak
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
- name: "DEBUG" - name: "GET client-secret for client <<{{ argo_client_id }}>> in realm <<{{ argo_realm_name }}>>"
debug: delegate_to: localhost
msg: "DEBUGGING: {{ client_secret.json.value }}" become: False
uri:
url: "{{ keycloak_server_url }}/auth/admin/realms/{{ argo_realm_name }}/clients/{{ id_of_client }}/client-secret"
method: GET
headers:
Content-Type: "application/json"
Authorization: "Bearer {{ access_token }}"
status_code: [200]
register: client_secret
when:
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
- name: "DEBUG"
debug:
msg: "DEBUGGING: {{ client_secret.json.value }}"
when:
- debug
- inventory_hostname == groups['kube_control_plane'][0]
tags:
- argo-cd
when: when:
- debug
- k8s_argocd_with_keycloak - k8s_argocd_with_keycloak
- inventory_hostname == groups['kube_control_plane'][0] # end of block statement
tags:
- argo-cd
- name: "Create namespace <{{ k8s_argocd_helm__release_namespace }}>" - name: "Create namespace <{{ k8s_argocd_helm__release_namespace }}>"
become: yes become: yes

@ -2,6 +2,8 @@
k8s_prometheus_helm__name: "prometheus" k8s_prometheus_helm__name: "prometheus"
cert_manager_dplmt: True
k8s_certmanager_helm__chart_ref: cert-manager k8s_certmanager_helm__chart_ref: cert-manager
k8s_certmanager_helm__chart_repo_url: https://charts.jetstack.io k8s_certmanager_helm__chart_repo_url: https://charts.jetstack.io
k8s_certmanager_helm__release_namespace: cert-manager k8s_certmanager_helm__release_namespace: cert-manager

@ -3,19 +3,17 @@
### tags: ### tags:
### cert-manager ### cert-manager
- name: Install cert-manager via helm - name: "Create namespace>"
become: yes become: yes
kubernetes.core.helm: kubernetes.core.k8s:
name: cert-manager name: "{{ k8s_certmanager_helm__release_namespace }}"
chart_ref: "{{ k8s_certmanager_helm__chart_ref }}" api_version: v1
chart_repo_url: "{{ k8s_certmanager_helm__chart_repo_url }}" kind: Namespace
release_namespace: "{{ k8s_certmanager_helm__release_namespace }}" state: present
create_namespace: yes
release_values: "{{ k8s_certmanager_helm__release_values }}"
when: when:
- inventory_hostname == groups['kube_control_plane'][0] - inventory_hostname == groups['kube_control_plane'][0]
tags: tags:
- cert-manager - namespace
- name: Create secret for digitalocean-dns - name: Create secret for digitalocean-dns
become: yes become: yes
@ -34,31 +32,47 @@
tags: tags:
- cert-manager - cert-manager
- name: Create ClusterIssuer for letsencrypt (prod/staging) - name: "Install cert-manager"
become: yes block:
kubernetes.core.k8s:
definition: - name: Install cert-manager via helm
api_version: cert-manager.io/v1 become: yes
kind: ClusterIssuer kubernetes.core.helm:
metadata: name: cert-manager
name: "letsencrypt-{{ item.key }}" chart_ref: "{{ k8s_certmanager_helm__chart_ref }}"
spec: chart_repo_url: "{{ k8s_certmanager_helm__chart_repo_url }}"
acme: release_namespace: "{{ k8s_certmanager_helm__release_namespace }}"
email: "{{ item.value.email }}" create_namespace: yes
server: "{{ item.value.server }}" release_values: "{{ k8s_certmanager_helm__release_values }}"
privateKeySecretRef:
name: issuer-account-key - name: Create ClusterIssuer for letsencrypt (prod/staging)
solvers: become: yes
- dns01: kubernetes.core.k8s:
digitalocean: definition:
tokenSecretRef: api_version: cert-manager.io/v1
name: digitalocean-dns kind: ClusterIssuer
key: access-token metadata:
selector: name: "letsencrypt-{{ item.key }}"
dnsZones: spec:
- 'smardigo.digital' acme:
loop: "{{ k8s_certmanager_helm__cluster_issuers | dict2items }}" email: "{{ item.value.email }}"
server: "{{ item.value.server }}"
privateKeySecretRef:
name: issuer-account-key
solvers:
- dns01:
digitalocean:
tokenSecretRef:
name: digitalocean-dns
key: access-token
selector:
dnsZones:
- 'smardigo.digital'
loop: "{{ k8s_certmanager_helm__cluster_issuers | dict2items }}"
# end of block statement
when: when:
- inventory_hostname == groups['kube_control_plane'][0] - inventory_hostname == groups['kube_control_plane'][0]
- cert_manager_dplmt
tags: tags:
- cert-manager - cert-manager

@ -3,4 +3,4 @@
# using kubespray default value => kube_pods_subnet # using kubespray default value => kube_pods_subnet
k8s_ccm__cluster_cidr: 10.233.64.0/18 k8s_ccm__cluster_cidr: 10.233.64.0/18
k8s_ccm__template: "hetzner-ccm-networks__v1.12.1.yaml.j2" k8s_ccm__template: "hetzner-ccm-networks__v1.13.0.yaml.j2"

@ -0,0 +1,106 @@
# NOTE: this release was tested against kubernetes v1.18.x
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hcloud-cloud-controller-manager
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hcloud-cloud-controller-manager
template:
metadata:
labels:
app: hcloud-cloud-controller-manager
spec:
serviceAccountName: cloud-controller-manager
dnsPolicy: Default
tolerations:
# this taint is set by all kubelets running `--cloud-provider=external`
# so we should tolerate it to schedule the cloud controller manager
- key: "node.cloudprovider.kubernetes.io/uninitialized"
value: "true"
effect: "NoSchedule"
- key: "CriticalAddonsOnly"
operator: "Exists"
# cloud controller manages should be able to run on masters
- key: "node-role.kubernetes.io/master"
effect: NoSchedule
operator: Exists
- key: "node-role.kubernetes.io/control-plane"
effect: NoSchedule
operator: Exists
- key: "node.kubernetes.io/not-ready"
effect: "NoSchedule"
hostNetwork: true
containers:
- image: hetznercloud/hcloud-cloud-controller-manager:v1.13.0
name: hcloud-cloud-controller-manager
command:
- "/bin/hcloud-cloud-controller-manager"
- "--cloud-provider=hcloud"
- "--leader-elect=false"
- "--allow-untagged-cloud"
- "--allocate-node-cidrs=true"
- "--cluster-cidr={{ k8s_ccm__cluster_cidr | default('10.244.0.0/16') }}"
resources:
requests:
cpu: 100m
memory: 50Mi
env:
- name: HCLOUD_METRICS_ENABLED
value: "true"
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: HCLOUD_TOKEN
valueFrom:
secretKeyRef:
name: hcloud-ccm
key: token
- name: HCLOUD_NETWORK
valueFrom:
secretKeyRef:
name: hcloud-ccm
key: network
priorityClassName: system-cluster-critical
---
apiVersion: v1
kind: Service
metadata:
name: 'hcloud-cloud-controller-manager-metrics'
namespace: kube-system
labels:
metrics: service-metrics
spec:
selector:
app: 'hcloud-cloud-controller-manager'
type: ClusterIP
ports:
- port: 8233
targetPort: 8233
protocol: TCP
name: http-metrics
Loading…
Cancel
Save