From 83c0473afe2ee2d7370eddeec3060dd8eeb03940 Mon Sep 17 00:00:00 2001 From: "Ketelsen, Sven" Date: Wed, 3 Nov 2021 12:57:39 +0000 Subject: [PATCH] =?UTF-8?q?DEV-231:=20Pagination=20f=C3=BCr=20dynamisches?= =?UTF-8?q?=20Inventory=20fehlt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inventory_plugins/netgo-hcloud.py | 69 +++++++++++++------ ...netgo-hcloud.yml => stage-netgo-hcloud.yml | 13 +++- stage_qa.netgo-hcloud.yml | 14 ---- 3 files changed, 58 insertions(+), 38 deletions(-) rename stage_dev.netgo-hcloud.yml => stage-netgo-hcloud.yml (63%) delete mode 100644 stage_qa.netgo-hcloud.yml diff --git a/inventory_plugins/netgo-hcloud.py b/inventory_plugins/netgo-hcloud.py index fc1b254..e0f9038 100644 --- a/inventory_plugins/netgo-hcloud.py +++ b/inventory_plugins/netgo-hcloud.py @@ -13,7 +13,7 @@ DOCUMENTATION = r""" requirements: - python >= 2.10 description: - - Reads inventory data from the Hetzner Cloud API. + - Reads inventory data from the Hetzner Cloud API. Server Groups are given by the label. Servers need two labels: service and stage. extends_documentation_fragment: - constructed - inventory_cache @@ -27,11 +27,11 @@ DOCUMENTATION = r""" required: true env: - name: HETZNER_CLOUD_TOKEN - stage: - description: The Hetzner Cloud stage. + label_selector: + description: Filter servers by this label selector. required: true env: - - name: HETZNER_CLOUD_STAGE + - name: HETZNER_LABEL_SELECTOR """ import json @@ -41,25 +41,43 @@ from ansible.module_utils.urls import open_url from ansible.module_utils._text import to_native from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable from ansible.release import __version__ - +from ansible.utils.display import Display class MyHcloudAPI: BASE = "https://api.hetzner.cloud" - def __init__(self, token, stage): + def __init__(self, token, label_selector): self.token = token - self.stage = stage + self.label_selector = label_selector def get_servers(self): - api_url = "%s/v1/servers?label_selector=stage=" % self.BASE + self.stage + display = Display() + try: - response = open_url( - api_url, - headers={"Authorization": "Bearer " + self.token}, - ) - result = json.loads(response.read()) - return result["servers"] + servers = [] + + # pagination with page_size per window, repeat until last page is reached + page = 1 + page_size = 20 + while page > 0: + api_url = "%s/v1/servers?label_selector=" % self.BASE + self.label_selector + "&per_page=" + str(page_size) + "&page=" + str(page) + #display.display(api_url) + + response = open_url( + api_url, + headers={"Authorization": "Bearer " + self.token}, + ) + result = json.loads(response.read()) + + servers += result["servers"] + #for server in result["servers"]: + # display.display(server["name"]) + + if result["meta"]["pagination"]["page"] == result["meta"]["pagination"]["last_page"]: + break + page += 1 + return servers except ValueError: raise AnsibleError("Incorrect JSON payload") except Exception as e: @@ -77,7 +95,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): def _read_servers_from_API(self): servers = MyHcloudAPI( - self.get_option("api_token"), self.get_option("stage") + self.get_option("api_token"), self.get_option("label_selector") ).get_servers() return servers @@ -115,22 +133,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): if cache_needs_update: self._cache[cache_key] = servers - self.populate(self.get_option("stage"), servers) + self.populate(servers) + + def populate(self, servers): + display = Display() - def populate(self, stage, servers): # Add a default top group 'hcloud' self.inventory.add_group(group="hcloud") - # Add a default top group 'stage_XYZ' - self.inventory.add_group(group="stage_" + stage) - for server in servers: serverName = server["name"] serverLabels = server["labels"] - serverService = serverLabels["service"] + serverStage = serverLabels["stage"] + serverService = serverLabels["service"] + + display.display("server:<" + serverName + ">, stage=<" + serverStage + ">, service=<" + serverService + ">") self.inventory.add_group(group=serverService) + self.inventory.add_group(group="stage_" + serverStage) self.inventory.add_host(serverName, group="hcloud") self.inventory.add_host(serverName, group=serverService) - self.inventory.add_host(serverName, group="stage_" + stage) + self.inventory.add_host(serverName, group="stage_" + serverStage) + +# TODO get the private server ip for the given stage +# self.inventory.set_variable(serverName, 'stage_server_ip', server["public_net"]["ipv4"]["ip"]) +# self.inventory.set_variable(serverName, 'stage_private_server_ip', ...) diff --git a/stage_dev.netgo-hcloud.yml b/stage-netgo-hcloud.yml similarity index 63% rename from stage_dev.netgo-hcloud.yml rename to stage-netgo-hcloud.yml index c6c67e0..c199909 100644 --- a/stage_dev.netgo-hcloud.yml +++ b/stage-netgo-hcloud.yml @@ -1,6 +1,15 @@ -plugin: netgo-hcloud +# dynamic inventory for hetzner which reads the stage variable from environment +# +# parameters: +# HETZNER_CLOUD_TOKEN := hetzner cloud api token +# HETZNER_LABEL_SELECTOR := the label selector to use (note: multiple selectors are not supported by rest api) +# (e.g. stage=dev) +# (e.g. service=prometheus) +# usage: +# export HETZNER_LABEL_SELECTOR='stage=dev' +# ansible-playbook -i stage-netgo-hcloud.yml ... -stage: dev +plugin: netgo-hcloud api_token: !vault | $ANSIBLE_VAULT;1.1;AES256 diff --git a/stage_qa.netgo-hcloud.yml b/stage_qa.netgo-hcloud.yml deleted file mode 100644 index 3400013..0000000 --- a/stage_qa.netgo-hcloud.yml +++ /dev/null @@ -1,14 +0,0 @@ -plugin: netgo-hcloud - -stage: qa - -api_token: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 61353364633862333333623664376666363464386461646131616465356534393935383132653730 - 3166333033326265356565646431326431336533656434620a626336333234303062323930363562 - 39656365663139343863646466343938313064653935646462333163343363656438656334316263 - 6132653139313530630a613037643634326231623733393432633163666139306138623837646231 - 61316362396566656430363539333137356464663038313334306634633431363237303736356134 - 61393836656634336231616338633265393638633763626632633066313461333736666436373338 - 39653732393838643261343730323666373736386662623236343133373762663630653735613130 - 36636230623930336465