feat: kubernetes bootstrap

- ccm
- ingress
- certmanager
- argo-cd
  {{ stage }}-kube-argocd.{{ domain }}
- prometheus
  {{ stage }}-kube-grafana.{{ domain }}

roles/kubernetes/apps/defaults/main.yml

@@ -1,11 +1,96 @@
 ---
+
+k8s_prometheus_helm__name: "prometheus"
+k8s_prometheus_helm__release_namespace: "monitoring"
+
+k8s_argocd_helm__name: "argo-cd"
+k8s_argocd_helm__release_namespace: "argo-cd"
+
+# https://github.com/grafana/helm-charts
+# https://github.com/prometheus-community/helm-charts
 k8s_prometheus_helm__release_values:
   grafana:
-    adminPassword: "8gsf8073g"
+    adminUser: "{{ grafana_admin_username }}"
+    adminPassword: "{{ grafana_admin_password }}"
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ip_whitelist | join(',') }}"
+      hosts:
+        - "{{ stage }}-kube-grafana.{{ domain }}"
+      tls:
+        - secretName: "{{ stage }}-kube-grafana-cert"
+          hosts:
+            - "{{ stage }}-kube-grafana.{{ domain }}"
+    deploymentStrategy:
+      type: Recreate
   kubeControllerManager:
     service:
       port: 10257
       targetPort: 10257
     serviceMonitor:
       https: true
-      insecureSkipVerify: true
+      insecureSkipVerify: true
+
+# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+k8s_argocd_helm__release_values:
+  controller:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+  repoServer:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+  server:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+        cert-manager.io/issue-temporary-certificate: "true"
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ip_whitelist | join(',') }}"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+        nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+        nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+      hosts:
+        - "{{ stage }}-kube-argocd.{{ domain }}"
+      tls:
+        - secretName: "{{ stage }}-kube-argocd-cert"
+          hosts:
+            - "{{ stage }}-kube-argocd.{{ domain }}"
+  dex:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"
+  redis:
+    metrics:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: "{{ k8s_argocd_helm__release_namespace }}"
+        additionalLabels:
+          release: "{{ k8s_prometheus_helm__name }}"

roles/kubernetes/apps/tasks/main.yml

@@ -6,10 +6,10 @@
 
 - name: Deploy kube-prometheus-stack inside monitoring namespace
   kubernetes.core.helm:
-    name: prometheus
+    name: "{{ k8s_prometheus_helm__name }}"
     chart_repo_url: "{{ k8s_prometheus_helm__chart_repo_url | default('https://prometheus-community.github.io/helm-charts') }}"
-    chart_ref: "{{ k8s_ingress_helm__chart_ref | default('kube-prometheus-stack') }}"
+    chart_ref: "{{ k8s_prometheus_helm__chart_ref | default('kube-prometheus-stack') }}"
-    release_namespace: "{{ k8s_prometheus_helm__release_namespace | default('monitoring') }}"
+    release_namespace: "{{ k8s_prometheus_helm__release_namespace }}"
     create_namespace: yes
     release_values: "{{ k8s_prometheus_helm__release_values }}"
   when: 
@@ -17,22 +17,15 @@
   tags:
     - prometheus
 
-- name: Add argo-cd chart repo
+- name: Deploy argo-cd inside argo-cd namespace
-  kubernetes.core.helm_repository:
-    name: argo-cd
-    repo_url: "https://argoproj.github.io/argo-helm"
-  when: 
-  - inventory_hostname == groups['kube-master'][0]
-  tags:
-    - argo-cd
-
-- name: Deploy Argo-CD inside argo-cd namespace
   kubernetes.core.helm:
-    name: argo-cd
+    name: "{{ k8s_argocd_helm__name }}"
-    chart_ref: argo-cd/argo-cd
+    chart_repo_url: "{{ k8s_argocd_helm__chart_repo_url | default('https://argoproj.github.io/argo-helm') }}"
-    release_namespace: argo-cd
+    chart_ref: "{{ k8s_argocd_helm__chart_ref | default('argo-cd') }}"
-    create_namespace: true
+    release_namespace: "{{ k8s_argocd_helm__release_namespace }}"
+    create_namespace: yes
+    release_values: "{{ k8s_argocd_helm__release_values }}"
   when: 
   - inventory_hostname == groups['kube-master'][0]
   tags:
-    - argo-cd
+    - argo-cd

roles/kubernetes/cloud-controller-manager/defaults/main.yml

@@ -1 +1,6 @@
 ---
+
+# using kubespray default value => kube_pods_subnet
+k8s_ccm__cluster_cidr: 10.233.64.0/18
+
+k8s_ccm__template: "hetzner-ccm-networks__v1.12.1.yaml.j2"

roles/kubernetes/cloud-controller-manager/tasks/main.yml

@@ -3,16 +3,6 @@
 ### tags:
 ###   ccm
 
-- name: Download Hetzner CCM
-  ansible.builtin.get_url:
-    url: https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.12.0/ccm-networks.yaml
-    dest: /tmp/ccm.yaml
-    mode: '0664'
-  when:
-  - inventory_hostname == groups['kube-master'][0]
-  tags:
-    - ccm
-
 - name: Create secret for Hetzner CCM
   kubernetes.core.k8s:
     definition:
@@ -28,16 +18,16 @@
       data:
         network: "{{ stage | string | b64encode }}"
         token: "{{ hetzner_authentication_token | string | b64encode }}"
-  when: 
+  when:
-  - inventory_hostname == groups['kube-master'][0]
+    - inventory_hostname == groups['kube-master'][0]
   tags:
     - ccm
 
-- name: Apply Hetzner CCM manifest to the cluster.
+- name: Applying CCM deployment
   kubernetes.core.k8s:
     state: present
-    src: /tmp/ccm.yaml
+    definition: "{{ lookup('template', k8s_ccm__template) }}"
   when:
-  - inventory_hostname == groups['kube-master'][0]
+    - inventory_hostname == groups['kube-master'][0]
   tags:
     - ccm

roles/kubernetes/cloud-controller-manager/templates/hetzner-ccm-networks__v1.12.0.yaml.j2

@@ -0,0 +1,86 @@
+# NOTE: this release was tested against kubernetes v1.18.x
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hcloud-cloud-controller-manager
+  namespace: kube-system
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hcloud-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        app: hcloud-cloud-controller-manager
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      serviceAccountName: cloud-controller-manager
+      dnsPolicy: Default
+      tolerations:
+        # this taint is set by all kubelets running `--cloud-provider=external`
+        # so we should tolerate it to schedule the cloud controller manager
+        - key: "node.cloudprovider.kubernetes.io/uninitialized"
+          value: "true"
+          effect: "NoSchedule"
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+        # cloud controller manages should be able to run on masters
+        - key: "node-role.kubernetes.io/master"
+          effect: NoSchedule
+        - key: "node-role.kubernetes.io/control-plane"
+          effect: NoSchedule
+        - key: "node.kubernetes.io/not-ready"
+          effect: "NoSchedule"
+      hostNetwork: true
+      containers:
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.0
+          name: hcloud-cloud-controller-manager
+          command:
+            - "/bin/hcloud-cloud-controller-manager"
+            - "--cloud-provider=hcloud"
+            - "--leader-elect=false"
+            - "--allow-untagged-cloud"
+            - "--allocate-node-cidrs=true"
+            - "--cluster-cidr={{ k8s_ccm__cluster_cidr | default('10.244.0.0/16') }}"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 50Mi
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HCLOUD_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: token
+            - name: HCLOUD_NETWORK
+              valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: network

roles/kubernetes/cloud-controller-manager/templates/hetzner-ccm-networks__v1.12.1.yaml.j2

@@ -0,0 +1,88 @@
+# NOTE: this release was tested against kubernetes v1.18.x
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hcloud-cloud-controller-manager
+  namespace: kube-system
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hcloud-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        app: hcloud-cloud-controller-manager
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      serviceAccountName: cloud-controller-manager
+      dnsPolicy: Default
+      tolerations:
+        # this taint is set by all kubelets running `--cloud-provider=external`
+        # so we should tolerate it to schedule the cloud controller manager
+        - key: "node.cloudprovider.kubernetes.io/uninitialized"
+          value: "true"
+          effect: "NoSchedule"
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+        # cloud controller manages should be able to run on masters
+        - key: "node-role.kubernetes.io/master"
+          effect: NoSchedule
+          operator: Exists
+        - key: "node-role.kubernetes.io/control-plane"
+          effect: NoSchedule
+          operator: Exists
+        - key: "node.kubernetes.io/not-ready"
+          effect: "NoSchedule"
+      hostNetwork: true
+      containers:
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.1
+          name: hcloud-cloud-controller-manager
+          command:
+            - "/bin/hcloud-cloud-controller-manager"
+            - "--cloud-provider=hcloud"
+            - "--leader-elect=false"
+            - "--allow-untagged-cloud"
+            - "--allocate-node-cidrs=true"
+            - "--cluster-cidr={{ k8s_ccm__cluster_cidr | default('10.244.0.0/16') }}"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 50Mi
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HCLOUD_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: token
+            - name: HCLOUD_NETWORK
+              valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: network

roles/kubernetes/ingress-controller/defaults/main.yml

@@ -1,32 +1,31 @@
 ---
-
 k8s_ingress_helm__release_values:
-      controller:
+  controller:
-        replicaCount: 2
+    replicaCount: 3
-        config:
+    config:
-          use-forwarded-headers: "true"
+      use-forwarded-headers: "true"
-          compute-full-forwarded-for: "true"
+      compute-full-forwarded-for: "true"
-          use-proxy-protocol: "true"
+      use-proxy-protocol: "true"
-          ssl-ciphers: "EECDH+AESGCM:EDH+AESGCM"
+      ssl-ciphers: "EECDH+AESGCM:EDH+AESGCM"
-          ssl-protocols: "TLSv1.3"
+      ssl-protocols: "TLSv1.3"
-        service:
+    service:
-          externalTrafficPolicy: Local
+      externalTrafficPolicy: Local
-          healthCheckNodePort: &healthchecknodeport  31066
+      healthCheckNodePort: &healthchecknodeport 31066
-          nodePorts:
+      nodePorts:
-            http: &httpnodeport 30473
+        http: &httpnodeport 30473
-            https: 30474
+        https: 30474
-          annotations:
+      annotations:
-            load-balancer.hetzner.cloud/location: nbg1
+        load-balancer.hetzner.cloud/type: "lb11"
-            load-balancer.hetzner.cloud/name: "{{ stage }}-ingress"
+        load-balancer.hetzner.cloud/location: nbg1
-            load-balancer.hetzner.cloud/type: "lb11"
+        load-balancer.hetzner.cloud/name: "{{ stage }}-ingress"
-            load-balancer.hetzner.cloud/disable-public-network: "true"
+        load-balancer.hetzner.cloud/disable-public-network: true
-            load-balancer.hetzner.cloud/network-zone: "eu-central"
+        load-balancer.hetzner.cloud/disable-private-ingress: true
-            load-balancer.hetzner.cloud/use-private-ip: "true"
+        load-balancer.hetzner.cloud/use-private-ip: true
-            load-balancer.hetzner.cloud/uses-proxyprotocol: "true"
+        load-balancer.hetzner.cloud/uses-proxyprotocol: true
-            load-balancer.hetzner.cloud/health-check-interval: "3s"
+        load-balancer.hetzner.cloud/health-check-interval: "3s"
-            load-balancer.hetzner.cloud/health-check-timeout: "1s"
+        load-balancer.hetzner.cloud/health-check-timeout: "1s"
-            load-balancer.hetzner.cloud/health-check-retries: 3
+        load-balancer.hetzner.cloud/health-check-retries: 3
-            load-balancer.hetzner.cloud/health-check-protocol: "tcp"
+        load-balancer.hetzner.cloud/health-check-protocol: "tcp"
-            load-balancer.hetzner.cloud/health-check-port: *httpnodeport
+        load-balancer.hetzner.cloud/health-check-port: *httpnodeport
-      defaultBackend:
+  defaultBackend:
-        enabled: true
+    enabled: true

roles/kubernetes/ingress-controller/files/hello-node__fullobjects.yaml

@@ -29,7 +29,7 @@ metadata:
   namespace: default
 spec:
   ports:
-  - port: 8080
+  - port: 80
     protocol: TCP
     targetPort: 8080
   selector:
@@ -56,7 +56,7 @@ spec:
           service:
             name: hello-node
             port:
-              number: 8080
+              number: 80
         path: /
         pathType: Prefix
   tls: