DEV-837 k8s bootstrap: added creation of load balancer and dns record

galaxy-requirements.yml

@@ -17,7 +17,7 @@ roles:
 - name: hetzner-ansible-hcloud
   src: git@git.dev-at.de:smardigo-hetzner/ansible/hetzner-ansible-roles/hetzner-ansible-hcloud-role.git
   scm: git
-  version: 0.0.2
+  version: 0.0.4
 - name: hetzner-ansible-common
   src: git@git.dev-at.de:smardigo-hetzner/ansible/hetzner-ansible-roles/hetzner-ansible-common-role.git
   scm: git

group_vars/all/plain.yml

@@ -56,11 +56,14 @@ http_s: "http{{ use_ssl | ternary('s', '', omit) }}"
 
 stage_server_domain: "{{ inventory_hostname }}.{{ domain }}"
 stage_server_url: "{{ http_s }}://{{ stage_server_domain }}"
+stage_kube_load_balancer: "{{ stage_kube }}-ingress"
 
 alertmanager_channel_smardigo: "#monitoring-{{ stage }}"
 
 hetzner_server_type: cx11
 hetzner_server_image: ubuntu-20.04
+hetzner_location: nbg1
+hetzner_load_balancer_type: lb11
 
 awx_ansible_user_name: "awx"
 awx_ansible_user_ssh_key_private: "{{ ansible_ssh_key_private_vault }}"

group_vars/all/services.yml

@@ -26,17 +26,22 @@ shared_service_kube_awx_hostname: "{{ stage_kube }}-awx.{{ domain_env }}"
 shared_service_kube_harbor_hostname: "{{ stage }}-harbor.{{ domain_env }}"
 shared_service_kube_jaeger_collector_hostname: "{{ stage_kube }}-jaeger-collector.{{ domain_env }}"
 shared_service_kube_prometheus_hostname: "{{ stage_kube }}-prometheus.{{ domain_env }}"
+
+# TODO make value available for plays with static inventory - by autodiscover_pre_tasks.yml
+shared_service_kube_loadbalancer_public_ip_not_available: "public loadbalancer ip not available"
+shared_service_kube_loadbalancer_public_ip: "{{ stage_public_ingress_loadbalancer_ip | default(shared_service_kube_loadbalancer_public_ip_not_available) }}"
 # TODO make value available for plays with static inventory - by autodiscover_pre_tasks.yml
-shared_service_kube_loadbalancer_ip_not_available: "private loadbalancer ip not available"
+shared_service_kube_loadbalancer_private_ip_not_available: "private loadbalancer ip not available"
-shared_service_kube_loadbalancer_ip: "{{ stage_private_ingress_loadbalancer_ip | default(shared_service_kube_loadbalancer_ip_not_available) }}"
+shared_service_kube_loadbalancer_private_ip: "{{ stage_private_ingress_loadbalancer_ip | default(shared_service_kube_loadbalancer_private_ip_not_available) }}"
+
 shared_service_additional_hosts:
   - name: "{{ shared_service_kube_argocd_hostname }}"
-    ip: "{{ shared_service_kube_loadbalancer_ip }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_awx_hostname }}"
-    ip: "{{ shared_service_kube_loadbalancer_ip }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_prometheus_hostname }}"
-    ip: "{{ shared_service_kube_loadbalancer_ip }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_jaeger_collector_hostname }}"
-    ip: "{{ shared_service_kube_loadbalancer_ip }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"
   - name: "{{ shared_service_kube_harbor_hostname }}"
-    ip: "{{ shared_service_kube_loadbalancer_ip }}"
+    ip: "{{ shared_service_kube_loadbalancer_private_ip }}"

hetzner_ssh_keys.yml

@@ -0,0 +1,15 @@
+---
+- name: 'upload ssh keys for root access to hetzner cloud'
+  hosts: '{{ host | default("all") }}'
+  gather_facts: false
+  connection: local
+
+  pre_tasks:
+    - name: "add ssh key for root access"
+      hetzner.hcloud.hcloud_ssh_key:
+        api_token: "{{ hetzner_authentication_ansible }}"
+        name: "{{ lookup('file', 'users/' + item + '/ssh.pub').split(' ') | last }}"
+        public_key: "{{ lookup('file', 'users/' + item + '/ssh.pub') }}"
+        state: "{{ ssh_key_state | default('present') }}"
+      loop: '{{ hetzner_ssh_keys }}'
+      run_once: true

inventory_plugins/netgo-hcloud.py

@@ -192,7 +192,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             loadbalancerName = loadbalancer["name"]
             loadbalancerLabels = loadbalancer["labels"]
             loadbalancerPublicIp = loadbalancer["public_net"]["ipv4"]["ip"]
+            if len(loadbalancer["private_net"]) > 0:
                 loadbalancerPrivateIp = loadbalancer["private_net"][0]["ip"]
+            else:
+                loadbalancerPrivateIp = '-'
 
             display.display("loadbalancer:<" + loadbalancerName + ">, publicIp=<" + loadbalancerPublicIp + ">, privateIp=<" + loadbalancerPrivateIp + ">")
 
@@ -236,5 +239,5 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             self.inventory.set_variable(serverName, 'stage_server_ip', serverPublicIp)
             self.inventory.set_variable(serverName, 'ansible_ssh_host', serverPublicIp)
             self.inventory.set_variable(serverName, 'stage_private_server_ip', serverPrivateIp)
-            self.inventory.set_variable(serverName, 'stage_ingress_loadbalancer_ip', loadbalancerPublicIp)
+            self.inventory.set_variable(serverName, 'stage_public_ingress_loadbalancer_ip', loadbalancerPublicIp)
             self.inventory.set_variable(serverName, 'stage_private_ingress_loadbalancer_ip', loadbalancerPrivateIp)

kubernetes.yml

@@ -1,12 +1,12 @@
 ---
+#############################################################
+#   Create default load balancer for kubernetes
+#############################################################
 
-# bootstraping kubernetes cluster
+- name: "Create default load balancer <{{ stage_kube_load_balancer }}>"
-
+  hosts: "{{ host | default('kube_control_plane') }}"
-- name: 'apply kubernetes setup to {{ host | default("all") }}'
+  gather_facts: false
-  hosts: '{{ host | default("kube_control_plane") }}'
+  connection: local
-  serial: "{{ serial_number | default(10) }}"
-  vars:
-    ansible_ssh_host: "{{ stage_server_domain }}"
 
   pre_tasks:
     - name: "Check if ansible version is at least {{ ansible_minimal_version }}"
@@ -17,12 +17,37 @@
       tags:
         - always
 
+  tasks:
+    - name: "Create default load balancer <{{ stage_kube_load_balancer }}>"
+      include_role:
+        name: hetzner-ansible-hcloud
+        tasks_from: configure_load_balancer
+      vars: 
+        - hetzner_load_balancer_name: "{{ stage_kube_load_balancer }}"
+        - hetzner_load_balancer_labels: "stage={{ stage }}"
+
+#############################################################
+#   Bootstrapping kubernetes cluster
+#############################################################
+
+- name: "apply kubernetes setup to {{ host | default('kube_control_plane') }}"
+  hosts: "{{ host | default('kube_control_plane') }}"
+  serial: "{{ serial_number | default(10) }}"
+  vars:
+    ansible_ssh_host: "{{ stage_server_domain }}"
+
+  pre_tasks:
     - name: "Import autodiscover pre-tasks"
       import_tasks: tasks/autodiscover_pre_tasks.yml
       tags:
         - always
 
   roles:
+    - role: hetzner-ansible-dns
+      vars:
+        record_data: "{{ shared_service_kube_loadbalancer_public_ip }}"
+        record_name: "{{ stage_kube_load_balancer }}"
+
     - { role: kubernetes/base }
 
     - role: kubernetes/cloud_controller_manager

roles/kubernetes/ingress_controller/defaults/main.yml

@@ -33,10 +33,10 @@ k8s_ingress_helm__release_values:
         https: 30474
       annotations:
         # see https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations
-        load-balancer.hetzner.cloud/type: "lb11"
+        load-balancer.hetzner.cloud/type: "{{ hetzner_load_balancer_type }}"
-        load-balancer.hetzner.cloud/location: nbg1
+        load-balancer.hetzner.cloud/location: "{{ hetzner_location }}"
-        load-balancer.hetzner.cloud/name: "{{ stage_kube }}-ingress"
+        load-balancer.hetzner.cloud/name: "{{ stage_kube_load_balancer }}"
-        load-balancer.hetzner.cloud/hostname: "{{ stage_kube }}-ingress"
+        load-balancer.hetzner.cloud/hostname: "{{ stage_kube_load_balancer }}"
         load-balancer.hetzner.cloud/disable-public-network: false
         load-balancer.hetzner.cloud/disable-private-ingress: true
         load-balancer.hetzner.cloud/use-private-ip: true

stage-dev

@@ -101,7 +101,6 @@ prometheus
 redis
 ubuntu_docker
 webdav
-test
 
 [all:children]
 stage_dev

tasks/autodiscover_pre_tasks.yml

@@ -116,22 +116,35 @@
 
 - name: "Reading hetzner loadbalancer infos for stage <{{ stage_kube }}> with pagination"
   set_fact:
-    shared_service_kube_loadbalancer_ip: "{{
+    shared_service_kube_loadbalancer_private_ip: "{{
        hetzner_loadbalancers.json.load_balancers
        | json_query(querystr1)
        | first
        | default([])
        | first
-       | default(shared_service_kube_loadbalancer_ip_not_available) }}"
+       | default(shared_service_kube_loadbalancer_private_ip_not_available) }}"
   vars:
-    querystr1: "[?name=='{{ stage_kube }}-ingress'].private_net[*].ip"
+    querystr1: "[?name=='{{ stage_kube_load_balancer }}'].private_net[*].ip"
+  delegate_to: 127.0.0.1
+  tags:
+    - always
+
+- name: "Reading hetzner loadbalancer infos for stage <{{ stage_kube }}> with pagination"
+  set_fact:
+    shared_service_kube_loadbalancer_public_ip: "{{
+       hetzner_loadbalancers.json.load_balancers
+       | json_query(querystr1)
+       | first
+       | default(shared_service_kube_loadbalancer_public_ip_not_available) }}"
+  vars:
+    querystr1: "[?name=='{{ stage_kube_load_balancer }}'].public_net.ipv4.ip"
   delegate_to: 127.0.0.1
   tags:
     - always
 
 - name: "Printing hetzner loadbalancer infos for stage <{{ stage_kube }}>"
   debug:
-    msg: "{{ shared_service_kube_loadbalancer_ip }}"
+    msg: "<public={{ shared_service_kube_loadbalancer_public_ip }}:private={{ shared_service_kube_loadbalancer_private_ip }}>"
   delegate_to: 127.0.0.1
   tags:
     - always