DEV-231: Pagination für dynamisches Inventory fehlt

inventory_plugins/netgo-hcloud.py

@@ -13,7 +13,7 @@ DOCUMENTATION = r"""
     requirements:
         - python >= 2.10
     description:
-        - Reads inventory data from the Hetzner Cloud API.
+        - Reads inventory data from the Hetzner Cloud API. Server Groups are given by the <service> label. Servers need two labels: service and stage.
     extends_documentation_fragment:
         - constructed
         - inventory_cache
@@ -27,11 +27,11 @@ DOCUMENTATION = r"""
             required: true
             env:
                 - name: HETZNER_CLOUD_TOKEN
-        stage:
+        label_selector:
-            description: The Hetzner Cloud stage.
+            description: Filter servers by this label selector.
             required: true
             env:
-                - name: HETZNER_CLOUD_STAGE
+                - name: HETZNER_LABEL_SELECTOR
 """
 
 import json
@@ -41,25 +41,43 @@ from ansible.module_utils.urls import open_url
 from ansible.module_utils._text import to_native
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
 from ansible.release import __version__
-
+from ansible.utils.display import Display
 
 class MyHcloudAPI:
 
     BASE = "https://api.hetzner.cloud"
 
-    def __init__(self, token, stage):
+    def __init__(self, token, label_selector):
         self.token = token
-        self.stage = stage
+        self.label_selector = label_selector
 
     def get_servers(self):
-        api_url = "%s/v1/servers?label_selector=stage=" % self.BASE + self.stage
+        display = Display()
+
         try:
-            response = open_url(
+            servers = []
-                api_url,
+
-                headers={"Authorization": "Bearer " + self.token},
+            # pagination with page_size per window, repeat until last page is reached
-            )
+            page = 1
-            result = json.loads(response.read())
+            page_size = 20
-            return result["servers"]
+            while page > 0:
+              api_url = "%s/v1/servers?label_selector=" % self.BASE + self.label_selector + "&per_page=" + str(page_size) + "&page=" + str(page)
+              #display.display(api_url)
+
+              response = open_url(
+                  api_url,
+                  headers={"Authorization": "Bearer " + self.token},
+              )
+              result = json.loads(response.read())
+
+              servers += result["servers"]
+              #for server in result["servers"]:
+              #  display.display(server["name"])
+
+              if result["meta"]["pagination"]["page"] == result["meta"]["pagination"]["last_page"]:
+                break
+              page += 1
+            return servers
         except ValueError:
             raise AnsibleError("Incorrect JSON payload")
         except Exception as e:
@@ -77,7 +95,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
     def _read_servers_from_API(self):
         servers = MyHcloudAPI(
-            self.get_option("api_token"), self.get_option("stage")
+            self.get_option("api_token"), self.get_option("label_selector")
         ).get_servers()
         return servers
 
@@ -115,22 +133,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         if cache_needs_update:
             self._cache[cache_key] = servers
 
-        self.populate(self.get_option("stage"), servers)
+        self.populate(servers)
+
+    def populate(self, servers):
+        display = Display()
 
-    def populate(self, stage, servers):
         # Add a default top group 'hcloud'
         self.inventory.add_group(group="hcloud")
 
-        # Add a default top group 'stage_XYZ'
-        self.inventory.add_group(group="stage_" + stage)
-
         for server in servers:
             serverName = server["name"]
             serverLabels = server["labels"]
-            serverService = serverLabels["service"] 
+            serverStage = serverLabels["stage"]
+            serverService = serverLabels["service"]
+
+            display.display("server:<" + serverName + ">, stage=<" + serverStage + ">, service=<" + serverService + ">")
 
             self.inventory.add_group(group=serverService)
+            self.inventory.add_group(group="stage_" + serverStage)
 
             self.inventory.add_host(serverName, group="hcloud")
             self.inventory.add_host(serverName, group=serverService)
-            self.inventory.add_host(serverName, group="stage_" + stage)
+            self.inventory.add_host(serverName, group="stage_" + serverStage)
+
+# TODO get the private server ip for the given stage
+#            self.inventory.set_variable(serverName, 'stage_server_ip', server["public_net"]["ipv4"]["ip"])
+#            self.inventory.set_variable(serverName, 'stage_private_server_ip', ...)

stage-netgo-hcloud.yml

@@ -1,6 +1,15 @@
-plugin: netgo-hcloud
+#  dynamic inventory for hetzner which reads the stage variable from environment
+#
+#  parameters:
+#    HETZNER_CLOUD_TOKEN := hetzner cloud api token
+#    HETZNER_LABEL_SELECTOR := the label selector to use (note: multiple selectors are not supported by rest api)
+#      (e.g. stage=dev)
+#      (e.g. service=prometheus)
+#  usage:
+#    export HETZNER_LABEL_SELECTOR='stage=dev'
+#    ansible-playbook -i stage-netgo-hcloud.yml ...
 
-stage: dev
+plugin: netgo-hcloud
 
 api_token: !vault |
           $ANSIBLE_VAULT;1.1;AES256

stage_qa.netgo-hcloud.yml

@@ -1,14 +0,0 @@
-plugin: netgo-hcloud
-
-stage: qa
-
-api_token: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          61353364633862333333623664376666363464386461646131616465356534393935383132653730
-          3166333033326265356565646431326431336533656434620a626336333234303062323930363562
-          39656365663139343863646466343938313064653935646462333163343363656438656334316263
-          6132653139313530630a613037643634326231623733393432633163666139306138623837646231
-          61316362396566656430363539333137356464663038313334306634633431363237303736356134
-          61393836656634336231616338633265393638633763626632633066313461333736666436373338
-          39653732393838643261343730323666373736386662623236343133373762663630653735613130
-          36636230623930336465