From c9c33c48c531fb1a1992430a44eb9604224d20fa Mon Sep 17 00:00:00 2001 From: MiTHRAL Date: Wed, 19 Aug 2026 15:38:29 -0400 Subject: [PATCH] Make CI validation stdlib-only for Python 3.9 runner container The node:20-bullseye runner image ships Python 3.9, which lacks tomllib (3.11+). Extract inline workflow validation into tools/validate_pack.py using regex-based parsing of the machine- generated packwiz TOML subset, and call it from the workflow. --- .gitea/workflows/build-pack.yaml | 86 +----------------- tools/validate_pack.py | 146 +++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 83 deletions(-) create mode 100644 tools/validate_pack.py diff --git a/.gitea/workflows/build-pack.yaml b/.gitea/workflows/build-pack.yaml index 43297df..4a94f14 100644 --- a/.gitea/workflows/build-pack.yaml +++ b/.gitea/workflows/build-pack.yaml @@ -29,66 +29,11 @@ jobs: - name: Integrity check working-directory: client_pack - run: | - python3 - <<'EOF' - import hashlib, os, re, sys, tomllib - index = tomllib.load(open("index.toml", "rb")) - indexed = {e["file"] for e in index["files"]} - mods = [f for f in os.listdir("mods") if f.endswith(".pw.toml")] - missing = [f for f in mods if f"mods/{f}" not in indexed] - if missing: - print("ERROR: pw.toml files missing from index.toml:", missing) - sys.exit(1) - extra = [f for f in indexed if f.startswith("mods/") and f not in {f"mods/{m}" for m in mods}] - if extra: - print("ERROR: index.toml entries missing from mods/:", extra) - sys.exit(1) - errors = 0 - for f in mods: - data = tomllib.load(open(f"mods/{f}", "rb")) - d = data.get("download") - if not d or not d.get("hash"): - print(f"ERROR: {f} missing [download] hash") - errors += 1 - continue - if not d.get("url") and not d.get("mode", "").startswith("metadata:"): - print(f"ERROR: {f} missing [download] url or metadata mode") - errors += 1 - if d.get("url", "").startswith("http://"): - print(f"ERROR: {f} uses insecure http url") - errors += 1 - hf = d.get("hash-format", "sha512") - if not re.fullmatch(r"[0-9a-f]+", d["hash"]): - print(f"ERROR: {f} hash not hex ({hf})") - errors += 1 - if errors: - sys.exit(1) - print(f"OK: {len(mods)} mods indexed, all have [download] url or metadata source") - EOF + run: python3 ../tools/validate_pack.py . integrity - name: Sanity grep download URLs working-directory: client_pack - run: | - python3 - <<'EOF' - import glob, sys, tomllib - bad = [] - url_count = 0 - meta_count = 0 - for f in sorted(glob.glob("mods/*.pw.toml")): - d = tomllib.load(open(f, "rb")).get("download", {}) - if d.get("url"): - url_count += 1 - elif d.get("mode", "").startswith("metadata:"): - meta_count += 1 - else: - bad.append(f) - if bad: - print("ERROR: mods without download.url or metadata mode:") - for f in bad: - print(" " + f) - sys.exit(1) - print(f"OK: {url_count} direct urls + {meta_count} metadata sources = all mods resolvable") - EOF + run: python3 ../tools/validate_pack.py . sanity - name: Export Modrinth pack working-directory: client_pack @@ -100,32 +45,7 @@ jobs: - name: Validate CurseForge pack completeness working-directory: client_pack - run: | - python3 - <<'EOF' - import json, os, re, sys, zipfile, tomllib - z = zipfile.ZipFile("../SciCraft-CurseForge.zip") - manifest = json.loads(z.read("manifest.json")) - pids = {f["projectID"] for f in manifest["files"]} - ov = {n.split("overrides/mods/")[1] for n in z.namelist() if n.startswith("overrides/mods/")} - missing = [] - for f in os.listdir("mods"): - if not f.endswith(".pw.toml"): - continue - data = tomllib.load(open(f"mods/{f}", "rb")) - if data.get("side") == "server": - continue - fn = data["filename"] - cf = data.get("update", {}).get("curseforge", {}).get("project-id") - if cf: - if cf not in pids: - missing.append(fn) - elif fn not in ov: - missing.append(fn) - if missing: - print("ERROR: mods missing from CurseForge export:", missing) - sys.exit(1) - print(f"OK: CurseForge export complete ({len(pids)} manifest files + {len(ov)} overrides)") - EOF + run: python3 ../tools/validate_pack.py . cf ../SciCraft-CurseForge.zip - name: Upload pack artifacts uses: actions/upload-artifact@v4 diff --git a/tools/validate_pack.py b/tools/validate_pack.py new file mode 100644 index 0000000..e9be1e5 --- /dev/null +++ b/tools/validate_pack.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Validate a packwiz pack: index integrity, download resolvability, CF export completeness. + +Stdlib-only (no tomllib) so it runs on Python 3.9 in CI containers. +Parses the limited, machine-generated TOML subset used by packwiz metadata. +""" + +import glob +import json +import os +import re +import sys +import zipfile + +PW_TOML = re.compile(r"^(\w[\w.-]*)\s*=\s*(?:\"([^\"]*)\"|([0-9]+))", re.M) +SECTION = re.compile(r"^\[\[?([\w.\-]+)\]\]?$", re.M) + + +def parse_toml(path): + """Parse the minimal TOML subset packwiz emits: scalars + sections + array-of-tables.""" + data = {} + current = data + array_section = None + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + m = SECTION.match(line) + if m: + name = m.group(1) + if line.startswith("[["): + key = name + array_section = key + data.setdefault(key, []) + current = {} + data[key].append(current) + else: + key = name.replace("-", "_").replace(".", "_") + if key not in data or not isinstance(data[key], dict): + data[key] = {} + current = data[key] + array_section = None + continue + m = PW_TOML.match(line) + if m: + key = m.group(1).replace("-", "_") + value = m.group(2) if m.group(2) is not None else m.group(3) + current[key] = value + return data + + +def check_integrity(pack_dir): + index = parse_toml(os.path.join(pack_dir, "index.toml")) + indexed = {e.get("file") for e in index.get("files", []) if e.get("file")} + mods = [os.path.basename(f) for f in glob.glob(os.path.join(pack_dir, "mods", "*.pw.toml"))] + missing = [f for f in mods if "mods/%s" % f not in indexed] + if missing: + print("ERROR: pw.toml files missing from index.toml:", missing) + sys.exit(1) + extra = [f for f in indexed if f.startswith("mods/") and f not in {"mods/%s" % m for m in mods}] + if extra: + print("ERROR: index.toml entries missing from mods/:", extra) + sys.exit(1) + errors = 0 + for f in mods: + data = parse_toml(os.path.join(pack_dir, "mods", f)) + d = data.get("download", {}) + if not d or not d.get("hash"): + print("ERROR: %s missing [download] hash" % f) + errors += 1 + continue + if not d.get("url") and not d.get("mode", "").startswith("metadata:"): + print("ERROR: %s missing [download] url or metadata mode" % f) + errors += 1 + if d.get("url", "").startswith("http://"): + print("ERROR: %s uses insecure http url" % f) + errors += 1 + hf = d.get("hash-format", "sha512") + if not re.fullmatch(r"[0-9a-f]+", d["hash"]): + print("ERROR: %s hash not hex (%s)" % (f, hf)) + errors += 1 + if errors: + sys.exit(1) + print("OK: %d mods indexed, all have [download] url or metadata source" % len(mods)) + + +def check_sanity(pack_dir): + bad = [] + url_count = 0 + meta_count = 0 + for f in sorted(glob.glob(os.path.join(pack_dir, "mods", "*.pw.toml"))): + d = parse_toml(f).get("download", {}) + if d.get("url"): + url_count += 1 + elif d.get("mode", "").startswith("metadata:"): + meta_count += 1 + else: + bad.append(f) + if bad: + print("ERROR: mods without download.url or metadata mode:") + for f in bad: + print(" " + f) + sys.exit(1) + print("OK: %d direct urls + %d metadata sources = all mods resolvable" % (url_count, meta_count)) + + +def check_cf(pack_dir, zip_path): + z = zipfile.ZipFile(zip_path) + manifest = json.loads(z.read("manifest.json")) + pids = {f["projectID"] for f in manifest["files"]} + ov = {n.split("overrides/mods/")[1] for n in z.namelist() if n.startswith("overrides/mods/")} + missing = [] + for f in glob.glob(os.path.join(pack_dir, "mods", "*.pw.toml")): + data = parse_toml(f) + if data.get("side") == "server": + continue + fn = data.get("filename") + cf = data.get("update_curseforge", {}).get("project_id") + if cf: + if int(cf) not in pids: + missing.append(fn) + elif fn not in ov: + missing.append(fn) + if missing: + print("ERROR: mods missing from CurseForge export:", missing) + sys.exit(1) + print("OK: CurseForge export complete (%d manifest files + %d overrides)" % (len(pids), len(ov))) + + +def main(): + if len(sys.argv) < 2: + print("usage: validate_pack.py [cf ]", file=sys.stderr) + sys.exit(2) + pack_dir = sys.argv[1] + cmd = sys.argv[2] if len(sys.argv) > 2 else "integrity" + if cmd == "cf": + check_cf(pack_dir, sys.argv[3]) + elif cmd == "sanity": + check_sanity(pack_dir) + else: + check_integrity(pack_dir) + + +if __name__ == "__main__": + main() \ No newline at end of file