DEV-253: digitalocean stuff - add droplet but not idempotentgit branch git branch plz check

README.md

@@ -59,11 +59,60 @@ Create/Start servers for stage-dev
 
 > with Poetry prefix with `poetry run`
 
+# ansible-builder
+[AWX](https://www.ansible.com/products/awx-project/) is used in smardigo setup do execute several playbooks due to user interaction within smardigo product. To finish ansible runs successfully we have to make sure every ansible dependency(ansible collection/role or pip package) is installed. therefore ansible-builder gave us the opportunity to configure the needed environment to exec plays successfully. 
+
+before every git-merge depending on changes concerning new ansible collections/roles and/or new pip-packages, please run the following:
+
+    cd ansible-builder/
+    ansible-builder build --tag awx-custom-ee:latest --container-runtime docker
+
+
+
+hopefully it will result in e.g.:
+
+    cd ansible-builder/
+    ATTENTION: to get more information plz set --verbosity 3	
+    ansible-builder build --tag awx-custom-ee:latest --container-runtime docker
+    Running command:
+      docker build -f context/Dockerfile -t awx-custom-ee:latest context
+    
+    Complete! The build context can be found at: /home/friedrich/sandbox/netgo_stuff/hetzner-ansible/ansible-builder/context
+    
+    real	2m56,131s
+    user	0m0,208s
+    sys	0m0,102s`
+ 
+you can start the docker container locally with e.g. :
+    docker run -it --rm --mount type=bind,source="$(pwd)"/,target=/gitrepo/   awx-custom-ee:latest /bin/bash
+
+
+after it, you are able to exec some ansbible-runs like:
+(please solve dependencies before starting to docker container)
+    cd /gitrepo
+    ansible-playbook -i stage-digitalocean external_monitoring.yml --ask-vault-password -u root --private-key sshkey_pw_less 
+    
+    export HETZNER_LABEL_SELECTOR='stage=dev'
+    ansible-playbook -i stage-netgo-hcloud.yml -l redis smardigo.yml --ask-vault-password  -u root --private-key sshkey_pw_less
+  
+
+if everything works fine, plz push the created docker container with:
+
+    docker login dev-docker-registry-01.smardigo.digital
+    docker tag XXXXXXXX dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee:latest
+    docker push dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee
+
+
+
+
+
+
+
 # TODO
 
 IPFire
   149.233.6.129 - eShelter
-  212.121.131.106 - Siemansdamm
+  212.121.131.106 - Siemensdamm
 
 Prometheus
   curl -X POST https://{{ prometheus-url }}/-/reload

ansible-builder/README.md

@@ -1,4 +1,6 @@
 # Execution Environment for AWX
 
     ansible-builder build --tag dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee:latest
+    docker login dev-docker-registry-01.smardigo.digital
+    docker tag XXXXXXXX dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee:latest
     docker push dev-docker-registry-01.smardigo.digital/awx/awx-custom-ee

external_monitoring.yml

@@ -0,0 +1,66 @@
+---
+- name: 'apply setup to {{ host | default("all") }}'
+  hosts: '{{ host | default("all") }}'
+  serial: "{{ serial_number | default(5) }}"
+  tasks:
+  - set_fact:
+      dev_prometheus_fqdn: 'dev-prometheus-01.{{ domain }}'
+      qa_prometheus_fqdn: 'qa-prometheus-01.{{ domain }}'
+
+  - set_fact:
+      dev_prometheus_ip: "{{ lookup('community.general.dig', dev_prometheus_fqdn ) }}"
+      qa_prometheus_ip: "{{ lookup('community.general.dig', qa_prometheus_fqdn ) }}"
+
+  - name: "Allow SSH in UFW"
+    ufw:
+      rule: limit
+      port: 22
+      proto: tcp
+
+  - name: "Allow port 9100 for node-exporter in UFW"
+    ufw:
+      rule: allow
+      port: 9100
+      proto: tcp
+      src: "{{ item }}"
+    loop:
+      - "{{ dev_prometheus_ip }}"
+      - "{{ qa_prometheus_ip }}"
+
+  - name: "Allow port 9115 for blackbox-exporter in UFW"
+    ufw:
+      rule: allow
+      port: 9115
+      proto: tcp
+      src: "{{ item }}"
+    loop:
+      - "{{ dev_prometheus_ip }}"
+      - "{{ qa_prometheus_ip }}"
+
+  - name: "Set firewall default policy"
+    ufw:
+      state: enabled
+      policy: reject
+
+  - name: "configure ssh_hardening"
+    include_role:
+      # include role from collection called 'devsec'
+      name: devsec.hardening.ssh_hardening
+      apply:
+        tags:
+          - ssh_hardening
+    tags:
+      - ssh_hardening
+
+  - name: "Install node-exporter via include_role"
+    include_role:
+      name:  cloudalchemy.node-exporter
+
+  - name: "Install blackbox-exporter via include_role"
+    include_role:
+      name: cloudalchemy.blackbox-exporter
+      apply:
+        tags:
+          - blackbox
+    tags:
+      - blackbox

galaxy-requirements.yml

@@ -4,6 +4,15 @@ roles:
   version: 1.7.0
 - name: idealista.prometheus_redis_exporter_role
   version: 2.1.0
+- name: cloudalchemy.node-exporter
+  version: 2.0.0
+  scm: git
+  src: https://github.com/cloudalchemy/ansible-node-exporter
+- name: cloudalchemy.blackbox-exporter
+  version: 1.0.0
+  src: https://github.com/cloudalchemy/ansible-blackbox-exporter
+  scm: git
+
 collections:
 - name: hetzner.hcloud
   version: 1.6.0
@@ -11,3 +20,8 @@ collections:
 - name: kubernetes.core
 - name: community.mysql
 - name: community.postgresql
+- name: community.digitalocean
+  version: 1.11.0
+- name: devsec.hardening
+  version: 7.12.0
+  src: https://github.com/dev-sec/ansible-collection-hardening

group_vars/all/plain.yml

@@ -1,6 +1,29 @@
 ---
 
 debug: false
+ssh_macs:
+  - umac-128-etm@openssh.com
+  - hmac-sha2-256-etm@openssh.com
+  - hmac-sha2-512-etm@openssh.com
+ssh_host_key_algorithms:
+  - rsa-sha2-512
+  - rsa-sha2-256
+  - ssh-ed25519
+ssh_kex:
+  - curve25519-sha256
+  - curve25519-sha256@libssh.org
+  - diffie-hellman-group-exchange-sha256
+  - diffie-hellman-group16-sha512
+  - diffie-hellman-group18-sha512
+  - diffie-hellman-group14-sha256
+ssh_ciphers:
+  - chacha20-poly1305@openssh.com
+  - aes128-ctr
+  - aes192-ctr
+  - aes256-ctr
+  - aes128-gcm@openssh.com
+  - aes256-gcm@openssh.com
+ssh_permit_root_login: 'yes'
 
 docker_enabled: true
 traefik_enabled: true
@@ -147,3 +170,9 @@ backup_directory: "/backups"
 
 #grafana_user_smardigo_login: "< see vault >"
 #grafana_user_smardigo_password: "< see vault >"
+
+blackbox_exporter_fqdn: "dev-blackbox-01.{{ domain }}"
+blackbox_http_2xx_targets:
+- 'https://{{ stage }}-keycloak-01.smardigo.digital/auth/'
+- 'https://{{ stage }}-management-01-connect.smardigo.digital/auth/'
+blackbox_http_2xx_additional_targets: []

group_vars/digitalocean/plain.yml

@@ -0,0 +1,7 @@
+---
+ansible_ssh_host: "{{ inventory_hostname }}.{{ domain }}"
+
+droplet_defaults:
+  size: s-1vcpu-1gb
+  region: fra1
+  image: ubuntu-20-04-x64

group_vars/service_blackbox/plain.yml

@@ -0,0 +1,3 @@
+---
+blackbox_exporter_cli_flags:
+  log.level: warn

hcloud_firewall.yml

@@ -0,0 +1,36 @@
+---
+- hosts: localhost
+  vars:
+    hcloud_firewall_objects:
+      -
+        name: monitoring-extern-https
+        state: present
+        rules:
+        - 
+          direction: in
+          protocol: tcp
+          port: '443'
+          source_ips:
+           - "{{ lookup('community.general.dig', 'dev-blackbox-01.smardigo.digital' ) }}/32"
+          destination_ips: []
+          description: null
+        apply_to:
+        -
+          type: label_selector
+          label_selector:
+            selector: 'service=connect'
+        -
+          type: label_selector
+          label_selector:
+            selector: 'service=keycloak'
+
+
+    
+  tasks:
+    - name: "Setup hcloud firewall via include_role" 
+      include_role:
+        name: hcloud
+        tasks_from: configure-firewall2 
+      loop: "{{ hcloud_firewall_objects }}"
+      loop_control:
+        loop_var: firewall_object

pip-requirements

@@ -1,5 +1,7 @@
 ansible
 ansible-core>=2.10
+ansible-builder
+dnspython
 hcloud>=1.16.0
 jmespath
 netaddr

provisioning.yml

@@ -57,3 +57,15 @@
       vars:
         hetzner_state: 'started'
       when: "'hcloud' in group_names"
+  tasks:
+    - name: "Create server in DO-cloud via include_tasks"
+      include_role:
+        name: _digitalocean
+        tasks_from: _create_server
+      vars:
+        droplet:
+          name: dev-blackbox-01
+          tags:
+            - service_blackbox
+            - stage_dev
+      when: "'digitalocean' in group_names"

roles/_digitalocean/tasks/_create_server.yml

@@ -0,0 +1,69 @@
+- name: "Create ssh key"
+  delegate_to: localhost
+  community.digitalocean.digital_ocean_sshkey:
+    oauth_token: "{{ digitalocean_authentication_token }}"
+    name: "{{ item }}"
+    ssh_pub_key: "{{ lookup('file', 'users/' + item + '/ssh.pub') }}"
+    state: present
+  register: result
+  loop: '{{ smardigo_plattform_users }}'
+
+- name: "Get fingerprints for ssh_keys"
+  delegate_to: localhost
+  community.digitalocean.digital_ocean_sshkey_info:
+    oauth_token: "{{ digitalocean_authentication_token }}"
+  register: do_sshkeys_found
+
+- set_fact:
+    droplet_combined: "{{ droplet_defaults | combine(droplet) }}"
+
+- block:
+  - set_fact:
+      tag_service: "{{ droplet.name | regex_search('[a-z]+-([a-z]+)-[0-9]+','\\1') | first | string }}" 
+      tag_stage: "{{ droplet.name | regex_search('([a-z]+)-[a-z]+-[0-9]+','\\1') | first | string }}"
+
+  - name: "Create new droplet"
+    delegate_to: localhost
+    community.digitalocean.digital_ocean_droplet:
+      oauth_token: "{{ digitalocean_authentication_token }}"
+      state: "{{ droplet_combined.state | default('present') }}"
+      name: "{{ droplet_combined.name }}"
+      size: "{{ droplet_combined.size }}"
+      region: "{{ droplet_combined.region }}"
+      image: "{{ droplet_combined.image }}"
+      wait_timeout: "{{ droplet_combined.wait_timeout | default(500) }}"
+      unique_name: "{{ droplet_combined.unique_name | default(True) }}"
+      ssh_keys: "{{ do_sshkeys_found.data | json_query(querystring) }}"
+    register: new_droplet
+    vars:
+      querystring: "[*].fingerprint"
+    
+  - name: "Tag new server"
+    delegate_to: localhost
+    community.digitalocean.digital_ocean_tag:
+      oauth_token: "{{ digitalocean_authentication_token }}"
+      name: "{{ item | replace('-','_') }}"
+      resource_id: "{{ new_droplet.data.droplet.id }}"
+      state: present
+    register: tag_response
+    loop: "{{ droplet.tags }}"
+
+  - name: "Set fact"
+    delegate_to: localhost
+    set_fact:
+      stage_server_ip: "{{ item }}"
+    loop: "{{ new_droplet.data | community.general.json_query(jsonquery_ipaddress) }}"
+    vars:
+      jsonquery_ipaddress: "droplet.networks.v4[?type=='public'].ip_address"
+  
+# TODO: abolish _digitalocean/tasks/domain.yml
+  - name: "Create dns record for droplet"
+    delegate_to: localhost
+    community.digitalocean.digital_ocean_domain_record:
+      oauth_token: "{{ digitalocean_authentication_token }}"
+      state: present
+      domain: "{{ domain }}"
+      type: A
+      name: "{{ new_droplet.data.droplet.name }}"
+      data: "{{ stage_server_ip }}"
+      force_update: yes

roles/hcloud/tasks/configure-firewall2.yml

@@ -0,0 +1,131 @@
+---
+- name: "Get all existing firewalls"
+  uri:
+    method: GET
+    url: "https://api.hetzner.cloud/v1/firewalls"
+    body_format: json
+    headers:
+      accept: application/json
+      authorization: Bearer {{ hetzner_authentication_token }}
+    status_code: [200]
+  register: hcloud_firewalls_all
+  delegate_to: 127.0.0.1
+  become: false
+
+- name: "Setting hetzner firewall pagination count: <{{ hcloud_firewalls_all.json.meta.pagination.last_page }}>"
+  set_fact:
+    total_server_pages: "{{ hcloud_firewalls_all.json.meta.pagination.last_page }}"
+  become: false
+  tags:
+    - always
+
+
+- name: "BLOCK << WITHOUT >> pagination"
+  block:
+  - set_fact:
+      lookup_fw_obj: "{{ hcloud_firewalls_all.json.firewalls | community.general.json_query(jsonquery_find_firewall_name) }}"
+    vars:
+       jsonquery_find_firewall_name: "[?name=='{{ firewall_object.name }}']"
+  when:
+    - total_server_pages == '1'
+
+
+- name: "<< WITH >> pagination"
+  block:
+  - name: "Get all existing firewalls"
+    uri:
+      method: GET
+      url: "https://api.hetzner.cloud/v1/firewalls?page={{ item }}"
+      body_format: json
+      headers:
+        accept: application/json
+        authorization: Bearer {{ hetzner_authentication_token }}
+      status_code: [200]
+    register: hcloud_firewalls_all
+    delegate_to: 127.0.0.1
+    become: false
+
+  - set_fact:
+      lookup_fw_obj: "{{ hcloud_firewalls_all.json.results | community.general.json_query(querystr1) | first | community.general.json_query(querystr2) | community.general.json_query(querystr2) }}"
+    vars:
+      querystr1: "[[*].json.firewalls]"
+      querystr2: "[?name=='{{ firewall_object.name }}']"
+  when:
+    - total_server_pages != '1'
+
+
+- name: "Create firewall rule for <<{{ firewall_object.name }}>>"
+  uri:
+    method: POST
+    url: "https://api.hetzner.cloud/v1/firewalls"
+    body_format: json
+    headers:
+      Content-Type:  application/json
+      authorization: Bearer {{ hetzner_authentication_token }}
+    body: "{{ firewall_object | to_json }}"
+    return_content: yes
+    status_code: [201]
+  delegate_to: 127.0.0.1
+  become: false
+  when:
+    - firewall_object.state == 'present'
+    - lookup_fw_obj | length == 0 
+
+- name: "Update firewall rule for <<{{ firewall_object.name }}>>"
+  uri:
+    method: PUT
+    url: "https://api.hetzner.cloud/v1/firewalls/{{ lookup_fw_obj.0.id }}"
+    body_format: json
+    headers:
+      Content-Type:  application/json
+      authorization: Bearer {{ hetzner_authentication_token }}
+    body: "{{ firewall_object | to_json }}"
+    return_content: yes
+    status_code: [200]
+  delegate_to: 127.0.0.1
+  become: false
+  when:
+    - firewall_object.state == 'present'
+    - lookup_fw_obj | length > 0 
+
+- name: "Delete firewall rule for <<{{ firewall_object.name }}>>"
+  block:
+
+  - set_fact:
+      deactivate_fw_obj:
+        remove_from: "{{ firewall_object.apply_to }}"
+
+  - name: "Step_1: Unset usage of firewall rule <<{{ firewall_object.name }}>>"
+    uri:
+      method: POST
+      url: "https://api.hetzner.cloud/v1/firewalls/{{ lookup_fw_obj.0.id }}/actions/remove_from_resources"
+      body_format: json
+      headers:
+        Content-Type:  application/json
+        authorization: Bearer {{ hetzner_authentication_token }}
+      body: "{{ deactivate_fw_obj | to_json }}"
+      return_content: yes
+      status_code: [201]
+    delegate_to: 127.0.0.1
+    become: false
+
+  - name: "Step_2: Delete firewall rule for <<{{ firewall_object.name }}>>"
+    uri:
+      method: DELETE
+      url: "https://api.hetzner.cloud/v1/firewalls/{{ lookup_fw_obj.0.id }}"
+      body_format: json
+      headers:
+        Content-Type:  application/json
+        authorization: Bearer {{ hetzner_authentication_token }}
+      return_content: yes
+      status_code: [204]
+    register: cleanup_firewall
+    delegate_to: 127.0.0.1
+    become: false
+    until: cleanup_firewall.status in [204]
+    retries: 15
+    delay: 10
+
+  when:
+    - firewall_object.state == 'absent'
+    - lookup_fw_obj | length > 0 

stage-digitalocean

@@ -0,0 +1,11 @@
+[blackbox]
+dev-blackbox-01
+
+[stage_dev:children]
+blackbox
+
+[all:children]
+stage_dev
+
+[digitalocean:children]
+stage_dev

tasks/autodiscover_pre_tasks.yml

@@ -28,6 +28,8 @@
   when:
     - total_server_pages == '1'
 
+- name: "Block << WITH >> pagination"
+  block:
   - name: "Reading hetzner server infos for stage <{{ stage }}> with pagination"
     uri:
       url: "https://api.hetzner.cloud/v1/servers?label_selector=stage={{ stage }}&page={{ item }}"
@@ -38,10 +40,6 @@
     with_sequence: start=1 end={{ total_server_pages }}
     delegate_to: 127.0.0.1
     become: false
-  tags:
-    - always
-  when:
-    - total_server_pages != '1'
 
   - name: "Reading hetzner server infos for stage <{{ stage }}> with pagination"
     set_fact:
@@ -51,10 +49,10 @@
       querystr2: "[]"
     delegate_to: 127.0.0.1
     become: false
-  tags:
-    - always
   when:
     - total_server_pages != '1'
+  tags:
+    - always
 
 - name: "Printing hetzner server infos for stage <{{ stage }}>"
   debug:

templates/prometheus/config/prometheus/alert.rules.j2

@@ -250,3 +250,13 @@ groups:
       identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
       summary: "Instance <{{ '{{' }} $labels.instance {{ '}}' }}> reporting backup duration took more than 30 minutes."
       description: "backup duration took more than 60 minutes."
+
+  - alert: endpoint down
+    expr: probe_success == 0
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      identifier: '{{ '{{' }} $labels.instance {{ '}}' }}'
+      summary: "Instance <{{ '{{' }} $labels.instance {{ '}}' }}> is down. plz check."
+      description: "checks if endpoint is reachable or not"

templates/prometheus/config/prometheus/prometheus.yml.j2

@@ -26,7 +26,7 @@ alerting:
 scrape_configs:
 
 ##############################################
-###              Apllications             ####
+###              Applications             ####
 ##############################################
 
   - job_name: 'prometheus'
@@ -336,6 +336,28 @@ scrape_configs:
         target_label:  __address__
         replacement:   $1
 
+  - job_name: 'node-exporter-external-monitoring(digitalocean)'
+    scheme: http
+    metrics_path: '/metrics'
+    static_configs:
+      - targets:
+        - '{{ blackbox_exporter_fqdn }}:9100'
+        labels:
+          env: {{ stage }}
+          project: smardigo
+    relabel_configs:
+      - source_labels: [job]
+        target_label:  job
+        replacement:   'node-exporter'
+      - source_labels: [__address__]
+        regex:         .*!(.*)
+        target_label:  instance
+        replacement:   $1
+      - source_labels: [__address__]
+        regex:         (.*)!.*
+        target_label:  __address__
+        replacement:   $1
+
 ##############################################
 ###               Databases               ####
 ##############################################
@@ -416,3 +438,22 @@ scrape_configs:
         regex:         (.*)!.*
         target_label:  __address__
         replacement:   $1
+##############################################
+###               blackbox                ####
+##############################################
+  - job_name: 'blackbox_smardigo'
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: {{ blackbox_http_2xx_targets + blackbox_http_2xx_additional_targets }}
+        labels:
+          env: {{ stage }}
+          project: smardigo
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: "{{ lookup('community.general.dig', blackbox_exporter_fqdn ) }}:9115"