fix: make update-platforms.py safe and self-contained; add catalog checks + CI

.github/workflows/checks.yml

@@ -0,0 +1,29 @@
+---
+name: Checks
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+
+permissions: {}
+
+jobs:
+  python:
+    name: Lint Python & validate catalog
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Install tools
+        run: pip install --quiet pyyaml ruff
+      - name: Ruff (scripts)
+        run: ruff check scripts/
+      - name: Ruff (apps)
+        run: ruff check apps/
+      - name: Validate catalog consistency
+        run: python3 scripts/check-catalog.py

README.md

@@ -6,11 +6,39 @@ Configuration of our testing Jenkins (https://testing.stackable.tech)
 
 This catalog contains the config data for the tests run in https://testing.stackable.tech.
 
-* [platforms.yaml](catalog/platforms.yaml) contains the definition of the test system vendors and platforms
+* [platforms.yaml](catalog/platforms.yaml) contains the definition of the test system vendors and platforms (do not hand edit this, see below)
 * [operator-tests.yaml](catalog/operator-tests.yaml) defines the integration tests for our operators
 
 See this [Nuclino page](https://app.nuclino.com/Stackable/Engineering/Configuring-Test-Jobs-4e993d84-19b4-4081-846d-738f9f38573d) for further information.
 
+### Updating the catalog
+
+**Do not hand edit `platforms.yaml`**
+
+[`scripts/update-platforms.py`](scripts/update-platforms.py) refreshes the Kubernetes versions in `platforms.yaml`.
+It fetches the latest versions from the Replicated and IONOS APIs, keeps the latest patch per minor version line, adds ARM variants where supported, and rewrites the file.
+
+Prerequisites:
+
+* Python 3 with `PyYAML` (`pip install pyyaml`)
+* [`replicated`](https://docs.replicated.com/reference/replicated-cli-installing) CLI, authenticated (`REPLICATED_API_TOKEN`)
+* [`ionosctl`](https://github.com/ionos-cloud/ionosctl) CLI, authenticated
+
+Run it from the repository root and review the diff before committing:
+
+```bash
+python3 scripts/update-platforms.py
+git diff catalog/platforms.yaml
+```
+
+[`scripts/check-catalog.py`](scripts/check-catalog.py) validates that `platforms.yaml` and `operator-tests.yaml` are consistent (every platform referenced by a test exists, every platform has a known provider).
+It exits non-zero on errors, so it is suitable for CI.
+Run it after editing either catalog file:
+
+```bash
+python3 scripts/check-catalog.py
+```
+
 ## Apps
 
 Under [apps/](apps/README.md), we maintain a bunch of Dockerized applications for the maintenance of Jenkins and to run the tests.

apps/src/modules/catalog.py

@@ -36,7 +36,6 @@ def read_platforms(logger):
     logger              logger (String-consuming function)
     """
     global platforms
-    platforms = hiyapyco.load("/platforms.yaml")["platforms"]
     platforms_yaml = hiyapyco.load("/platforms.yaml")
     if "platforms" not in platforms_yaml:
         logger("platforms.yaml does not contain platforms.")

catalog/platforms.yaml

@@ -9,11 +9,11 @@ platforms:
       instance-type: r1.xlarge
       node-count: 1
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
+      - 1.36.1
+      - 1.35.5
+      - 1.34.8
+      - 1.33.12
       - 1.32.11
-      - 1.31.14
   - id: replicated-kind-arm
     name: kind on replicated.com - ARM
     provider: replicated
@@ -23,11 +23,11 @@ platforms:
       instance-type: r1a.xlarge
       node-count: 1
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
+      - 1.36.1
+      - 1.35.5
+      - 1.34.8
+      - 1.33.12
       - 1.32.11
-      - 1.31.14
   - id: replicated-rke2
     name: RKE2 on replicated.com
     provider: replicated
@@ -37,10 +37,10 @@ platforms:
       instance-type: r1.large
       node-count: 3
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
-      - 1.32.11
+      - 1.35.6
+      - 1.34.9
+      - 1.33.13
+      - 1.32.13
   - id: replicated-rke2-arm
     name: RKE2 on replicated.com - ARM
     provider: replicated
@@ -50,10 +50,10 @@ platforms:
       instance-type: r1a.large
       node-count: 3
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
-      - 1.32.11
+      - 1.35.6
+      - 1.34.9
+      - 1.33.13
+      - 1.32.13
   - id: replicated-k3s
     name: k3s on replicated.com
     provider: replicated
@@ -63,10 +63,11 @@ platforms:
       instance-type: r1.large
       node-count: 3
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
-      - 1.32.11
+      - 1.36.2
+      - 1.35.6
+      - 1.34.9
+      - 1.33.13
+      - 1.32.13
   - id: replicated-k3s-arm
     name: k3s on replicated.com - ARM
     provider: replicated
@@ -76,10 +77,11 @@ platforms:
       instance-type: r1a.large
       node-count: 3
     versions:
-      - 1.35.0
-      - 1.34.3
-      - 1.33.7
-      - 1.32.11
+      - 1.36.2
+      - 1.35.6
+      - 1.34.9
+      - 1.33.13
+      - 1.32.13
   - id: replicated-azure
     name: Azure AKS on replicated.com
     provider: replicated
@@ -89,9 +91,10 @@ platforms:
       instance-type: Standard_D4S_v5
       node-count: 3
     versions:
+      - "1.36"
+      - "1.35"
       - "1.34"
       - "1.33"
-      - "1.32"
   - id: replicated-azure-arm
     name: Azure AKS on replicated.com - ARM
     provider: replicated
@@ -101,9 +104,10 @@ platforms:
       instance-type: Standard_D4ps_v5
       node-count: 3
     versions:
+      - "1.36"
+      - "1.35"
       - "1.34"
       - "1.33"
-      - "1.32"
   - id: replicated-aws
     name: Amazon AWS EKS on replicated.com
     provider: replicated
@@ -113,11 +117,11 @@ platforms:
       instance-type: m6i.large
       node-count: 3
     versions:
+      - "1.36"
       - "1.35"
       - "1.34"
       - "1.33"
       - "1.32"
-      - "1.31"
   - id: replicated-aws-arm
     name: Amazon AWS EKS on replicated.com - ARM
     provider: replicated
@@ -127,11 +131,11 @@ platforms:
       instance-type: m7g.large
       node-count: 3
     versions:
+      - "1.36"
       - "1.35"
       - "1.34"
       - "1.33"
       - "1.32"
-      - "1.31"
   - id: replicated-gke
     name: Google GKE on replicated.com
     provider: replicated
@@ -141,6 +145,8 @@ platforms:
       instance-type: e2-standard-2
       node-count: 3
     versions:
+      - "1.36"
+      - "1.35"
       - "1.34"
       - "1.33"
       - "1.32"
@@ -153,6 +159,8 @@ platforms:
       instance-type: t2a-standard-2
       node-count: 3
     versions:
+      - "1.36"
+      - "1.35"
       - "1.34"
       - "1.33"
       - "1.32"
@@ -164,7 +172,7 @@ platforms:
       distribution: openshift
       instance-type: r1.large
       node-count: 3
-    versions: &id001
+    versions:
       - 4.22.0-okd
       - 4.21.0-okd
       - 4.20.0-okd
@@ -178,7 +186,12 @@ platforms:
       distribution: openshift
       instance-type: r1a.large
       node-count: 3
-    versions: *id001
+    versions:
+      - 4.22.0-okd
+      - 4.21.0-okd
+      - 4.20.0-okd
+      - 4.19.0-okd
+      - 4.18.0-okd
   - id: ionos-k8s
     name: IONOS managed K8s
     provider: ionos
@@ -190,9 +203,9 @@ platforms:
       node-count: 3
       ram: 8192
     versions:
+      - 1.34.2
       - 1.33.3
-      - 1.32.6
-      - 1.31.10
+      - 1.32.7
   - id: replicated-oke
     name: OKE on replicated.com
     provider: replicated
@@ -202,9 +215,9 @@ platforms:
       instance-type: VM.Standard3.Flex.2
       node-count: 3
     versions:
+      - 1.35.2
       - 1.34.2
-      - 1.33.1
-      - 1.32.10
+      - 1.33.10
   - id: replicated-oke-arm
     name: OKE on replicated.com - ARM
     provider: replicated
@@ -214,9 +227,9 @@ platforms:
       instance-type: VM.Standard.A1.Flex.2
       node-count: 3
     versions:
+      - 1.35.2
       - 1.34.2
-      - 1.33.1
-      - 1.32.10
+      - 1.33.10
 providers:
   - id: replicated
     name: replicated.com