Scripts and CI

.github/workflows/ci.yml

@@ -0,0 +1,107 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  typos:
+    name: "Typos"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check typos
+        uses: crate-ci/typos@master
+        with:
+          config: ./tools/typos.toml
+
+  typstyle:
+    name: "Typst formatting"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Download Typstyle"
+        run: |
+          wget -q "https://github.com/Enter-tainer/typstyle/releases/download/v0.12.14/typstyle-x86_64-unknown-linux-musl"
+          chmod +x typstyle-x86_64-unknown-linux-musl
+
+      - name: Check typst formatting
+        run: |
+          find . -name "*.typ" -type f -print0 | xargs -0 \
+            ./typstyle-x86_64-unknown-linux-musl --check
+
+  build:
+    needs:
+      - typos
+      - typstyle
+
+    name: "Build"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      # Could instead install `texlive-full`, but that takes ~20 minutes.
+      # We'll specify the packages we need manually.
+      - name: "Install TeXLive"
+        run: |
+          sudo apt update
+          DEBIAN_FRONTEND=noninteractive \
+            sudo apt install --yes \
+            texlive texlive-xetex \
+            texlive-games texlive-fonts-extra texlive-latex-extra \
+            texlive-pictures texlive-pstricks \
+            python3-requests
+
+      # Typst isn't packaged, and manual download gives us
+      # more control anyway.
+      - name: "Download Typst"
+        run: |
+          wget -q "https://github.com/typst/typst/releases/download/v0.12.0/typst-x86_64-unknown-linux-musl.tar.xz"
+          tar -xf "typst-x86_64-unknown-linux-musl.tar.xz"
+          mv "typst-x86_64-unknown-linux-musl/typst" .
+          rm "typst-x86_64-unknown-linux-musl.tar.xz"
+          rm -dr "typst-x86_64-unknown-linux-musl"
+
+      # Builds all handouts, LaTeX and Typst
+      - name: "Build handouts"
+        run: TYPST_PATH="$(pwd)/typst" python tools/scripts/build.py
+
+      # Upload logs, even if build fails.
+      # LaTeX stdout/stderr isn't always helpful.
+      - name: "Save LaTeX logs"
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: "LaTeX logs"
+          path: "**/*.log"
+          retention-days: 14
+
+      # Upload build output
+      - name: "Save output"
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Build output"
+          path: "output/*"
+          retention-days: 7
+
+      - name: "Publish package (hash)"
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        run: |
+          PUBLISH_USER="${{ secrets.PUBLISH_USER }}" \
+          PUBLISH_KEY="${{ secrets.PUBLISH_KEY }}" \
+          VERSION="${{ github.sha }}" \
+          PACKAGE="${{ vars.PACKAGE }}" \
+            python tools/scripts/publish.py
+
+      - name: "Publish package (latest)"
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        run: |
+          PUBLISH_USER="${{ secrets.PUBLISH_USER }}" \
+          PUBLISH_KEY="${{ secrets.PUBLISH_KEY }}" \
+          VERSION="latest" \
+          PACKAGE="${{ vars.PACKAGE }}" \
+            python tools/scripts/publish.py

tools/meta.toml

@@ -0,0 +1,23 @@
+# This is a sample `meta.toml`.
+# A copy of this file should exist in every handout directory.
+#
+# All keys are required.
+
+[metadata]
+title = "title of this handout"
+
+
+[publish]
+# Should we publish this handout?
+# If `false`, no part of this handout is published.
+# (useful for drafts)
+handout = true
+
+# Should we publish an "instructor's" version of this handout?
+# This has no effect if `publish.handout == false`.
+#
+# If `true`, publish a version of this handout with solutions.
+# If `false`, do not publish a handout with solutions.
+#
+# Set this to `false` if solutions have not been written.
+solutions = true

tools/scripts/build.py

@@ -0,0 +1,302 @@
+from typing import TypedDict
+from pathlib import Path
+import subprocess
+import tomllib
+import shutil
+import json
+import os
+
+# TODO:
+# list handouts without solutions
+# list handouts that are not published
+# lint: always commit without `nosolutions`
+# use latexmk
+
+ROOT: Path = Path(os.getcwd())
+
+### CONFIGURATION
+OUT_DIR: Path = ROOT / "output"
+TYPST_PATH: str = "typst"
+XETEX_PATH: str = "xelatex"
+### END CONFIGURATION
+
+# Allow path override
+_env = os.environ.get("TYPST_PATH")
+if isinstance(_env, str):
+	TYPST_PATH = _env
+
+
+def log(msg):
+	print(f"[BUILD.PY] {msg}")
+
+
+log(f"Running in {ROOT}")
+if not ROOT.is_dir():
+	log("Root is not a directory, cannot continue")
+	exit(1)
+
+log(f"Output dir is {OUT_DIR}")
+if OUT_DIR.exists():
+	log("Output dir exists, removing")
+	shutil.rmtree(OUT_DIR)
+
+OUT_DIR.mkdir(parents=True)
+
+
+IndexEntry = TypedDict(
+	"IndexEntry",
+	{"title": str, "group": str, "handout_file": str, "solutions_file": str | None},
+)
+
+
+MetaToml = TypedDict(
+	"MetaToml", {"title": str, "publish_handout": bool, "publish_solutions": bool}
+)
+
+
+def sanitize_file_name(file: str) -> str:
+	return file.replace("?", "")
+
+
+def read_meta_toml(file: Path) -> MetaToml:
+	with file.open("rb") as f:
+		base = tomllib.load(f)
+
+	meta = base.get("metadata")
+	if not isinstance(meta, dict):
+		log("Invalid meta.toml: `metadata` should be a table")
+		exit(1)
+
+	title = meta.get("title")
+	if not isinstance(title, str):
+		log("Invalid meta.toml: `title` should be a string")
+		exit(1)
+
+	pub = base.get("publish")
+	if not isinstance(pub, dict):
+		log("Invalid meta.toml: `publish` should be a table")
+		exit(1)
+
+	pub_handout = pub.get("handout")
+	if not isinstance(pub_handout, bool):
+		log("Invalid meta.toml: `publish.handout` should be a boolean")
+		exit(1)
+
+	pub_solutions = pub.get("solutions")
+	if not isinstance(pub_solutions, bool):
+		log("Invalid meta.toml: `publish.solutions` should be a boolean")
+		exit(1)
+
+	return {
+		"title": title,
+		"publish_handout": pub_handout,
+		"publish_solutions": pub_solutions,
+	}
+
+
+def log_error(res):
+	stdout = res.stdout.decode("utf-8").strip()
+	stderr = res.stderr.decode("utf-8").strip()
+
+	log(f"Build failed with code {res.returncode}")
+
+	if stdout == "":
+		log("Stdout: <empty>")
+	else:
+		log("Stdout:")
+		print(stdout)
+
+	if stderr == "":
+		log("Stderr: <empty>")
+	else:
+		log("Stderr:")
+		print(stderr)
+
+	exit(1)
+
+
+def build_typst(source_dir: Path, out_subdir: Path) -> IndexEntry | None:
+	if not (source_dir / "main.typ").is_file():
+		# log(f"No main.typ, skipping {source_dir}")
+		return None
+
+	meta_path = source_dir / "meta.toml"
+	if not meta_path.is_file():
+		log(f"No meta.toml, skipping {source_dir}")
+		return None
+	meta = read_meta_toml(meta_path)
+
+	handout_file = sanitize_file_name(f"{meta['title']}.pdf")
+	solutions_file = sanitize_file_name(f"{meta['title']}.sols.pdf")
+
+	# Do nothing if not published
+	if not meta["publish_handout"]:
+		return None
+
+	# Build handout
+	log(f"Building typst (handout)  : {source_dir}")
+
+	out = OUT_DIR / out_subdir
+	out.mkdir(parents=True, exist_ok=True)
+
+	res = subprocess.run(
+		[
+			TYPST_PATH,
+			"compile",
+			"--ignore-system-fonts",
+			"main.typ",
+			"--input",
+			"show_solutions=false",
+			f"{out}/{handout_file}",
+		],
+		cwd=source_dir,
+		stdout=subprocess.PIPE,
+		stderr=subprocess.PIPE,
+	)
+
+	if res.returncode != 0:
+		log_error(res)
+
+	# Build solutions
+	if meta["publish_solutions"]:
+		log(f"Building typst (solutions): {source_dir}")
+		res = subprocess.run(
+			[
+				TYPST_PATH,
+				"compile",
+				"--ignore-system-fonts",
+				"main.typ",
+				f"{out}/{solutions_file}",
+			],
+			cwd=source_dir,
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE,
+		)
+
+		if res.returncode != 0:
+			log_error(res)
+
+	return {
+		"title": meta["title"],
+		"group": str(out_subdir),
+		"handout_file": str(out / handout_file),
+		"solutions_file": (
+			str(out / solutions_file) if meta["publish_solutions"] else None
+		),
+	}
+
+
+def build_xetex(source_dir: Path, out_subdir: Path) -> IndexEntry | None:
+	if not (source_dir / "main.tex").is_file():
+		# log(f"No main.tex, skipping {source_dir}")
+		return None
+
+	meta_path = source_dir / "meta.toml"
+	if not meta_path.is_file():
+		log(f"No meta.toml, skipping {source_dir}")
+		return None
+	meta = read_meta_toml(meta_path)
+
+	handout_file = sanitize_file_name(f"{meta['title']}.pdf")
+	solutions_file = sanitize_file_name(f"{meta['title']}.sols.pdf")
+
+	# Do nothing if not published
+	if not meta["publish_handout"]:
+		return None
+
+	# Build handout
+	log(f"Building xetex (handout)  : {source_dir}")
+
+	out = OUT_DIR / out_subdir
+	out.mkdir(parents=True, exist_ok=True)
+
+	res = subprocess.run(
+		[
+			XETEX_PATH,
+			"-interaction=batchmode",
+		],
+		input=b"\\def\\argNoSolutions{1}\\input{main.tex}",
+		cwd=source_dir,
+		stdout=subprocess.PIPE,
+		stderr=subprocess.PIPE,
+	)
+
+	try:
+		shutil.copy(source_dir / "main.pdf", f"{out}/{handout_file}")
+	except Exception as e:
+		log(f"Error: {e}")
+		log_error(res)
+
+	if res.returncode != 0:
+		log_error(res)
+
+	# Build solutions
+	if meta["publish_solutions"]:
+		log(f"Building xetex (solutions): {source_dir}")
+		res = subprocess.run(
+			[
+				XETEX_PATH,
+				"-interaction=batchmode",
+			],
+			input=b"\\def\\argYesSolutions{1}\\input{main.tex}",
+			cwd=source_dir,
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE,
+		)
+
+		try:
+			shutil.copy(source_dir / "main.pdf", f"{out}/{solutions_file}")
+		except Exception as e:
+			log(f"Error: {e}")
+			log_error(res)
+
+		if res.returncode != 0:
+			log_error(res)
+
+	return {
+		"title": meta["title"],
+		"group": str(out_subdir),
+		"handout_file": str(out / handout_file),
+		"solutions_file": (
+			str(out / solutions_file) if meta["publish_solutions"] else None
+		),
+	}
+
+
+# Try to build handouts in all subdirs of `base`,
+# skipping those that do not contain a handout.
+#
+# This method does _not_ recurse into subdirectories.
+def build_dir(base: str, out_sub: str, index: list[IndexEntry]):
+	builders = [build_typst, build_xetex]
+
+	for d_str in os.listdir(base):
+		d = base / Path(d_str)
+		if not d.is_dir():
+			continue
+
+		# Try each builder,
+		# stopping at the first success
+		done = False
+		for builder in builders:
+			res = builder(d, Path(out_sub))
+			if res is not None:
+				# Check for duplicate titles
+				for i in index:
+					if i["title"] == res["title"]:
+						log(f'Duplicate title "{i["title"]}", cannot continue')
+						exit(1)
+				# Save to index and exit
+				index.append(res)
+				done = True
+			if done:
+				break
+
+
+index: list[IndexEntry] = []
+build_dir("src/Advanced", "Advanced", index)
+build_dir("src/Intermediate", "Intermediate", index)
+build_dir("src/Warm-Ups", "Warm-Ups", index)
+
+with open(OUT_DIR / "index.json", "w") as f:
+	f.write(json.dumps(index))

tools/scripts/publish.py

@@ -0,0 +1,95 @@
+# Publish the output of `build.py`
+# as a Gitea package.
+
+from pathlib import Path
+import requests
+import json
+import os
+import re
+
+URL = "https://git.betalupi.com"
+USER = os.environ["PUBLISH_USER"]
+PACKAGE = os.environ["PACKAGE"]
+VERSION = os.environ["VERSION"]
+AUTH = requests.auth.HTTPBasicAuth(USER, os.environ["PUBLISH_KEY"])
+
+ROOT: Path = Path(os.getcwd())
+SRC_DIR: Path = ROOT / "output"
+
+
+def log(msg):
+	print(f"[PUBLISH.PY] {msg}")
+
+
+log(f"Version is {VERSION}")
+log(f"Package is {PACKAGE}")
+log(f"Running in {ROOT}")
+if not ROOT.is_dir():
+	log("Root is not a directory, cannot continue")
+	exit(1)
+
+log(f"Source dir is {SRC_DIR}")
+if not SRC_DIR.exists():
+	log("Source dir doesn't exist, cannot continue")
+	exit(1)
+
+
+def del_package():
+	log(f"Deleting package {PACKAGE}/{VERSION}")
+	res = requests.delete(
+		f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}",
+		auth=AUTH,
+	)
+	if res.status_code != 204 and res.status_code != 404:
+		log(f"Deletion failed with code {res.status_code}")
+
+
+# Delete if already exists
+# (important for the `latest` package)
+del_package()
+
+
+def upload(data, target: str):
+	target = re.sub("[^A-Za-z0-9_. -]+", "", target)
+
+	res = requests.put(
+		f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}/{target}",
+		auth=AUTH,
+		data=data,
+	)
+
+	if res.status_code != 201:
+		log(f"Upload failed with code {res.status_code}")
+		del_package()  # Do not keep partial package if upload fails
+		exit(1)
+
+	return f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}/{target}"
+
+
+index_file = SRC_DIR / "index.json"
+with index_file.open("r") as f:
+	index = json.load(f)
+
+new_index = []
+for handout in index:
+	title = handout["title"]
+	group = handout["group"]
+	h_file = SRC_DIR / handout["handout_file"]
+	s_file = handout["solutions_file"]
+	if s_file is not None:
+		s_file = SRC_DIR / s_file
+	log(f"Uploading {title}")
+
+	h_url = None
+	s_url = None
+
+	h_url = upload(h_file.open("rb").read(), f"{group} - {title}.pdf")
+	if s_file is not None:
+		log(f"Uploading {title} solutions")
+		s_url = upload(s_file.open("rb").read(), f"{group} - {title}.sols.pdf")
+
+	new_index.append(
+		{"title": title, "group": group, "handout": h_url, "solutions": s_url}
+	)
+
+upload(json.dumps(new_index), "index.json")

tools/scripts/ruff.toml

@@ -0,0 +1,18 @@
+exclude = ["venv"]
+line-length = 88
+indent-width = 4
+target-version = "py39"
+include = ["scripts/**/*.py"]
+
+[lint]
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+fixable = ["ALL"]
+unfixable = []
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+quote-style = "double"
+indent-style = "tab"
+skip-magic-trailing-comma = false
+line-ending = "lf"

tools/typos.toml

@@ -0,0 +1,11 @@
+[default]
+extend-words."LSAT" = "LSAT"
+extend-words."ket" = "ket"
+
+extend-ignore-re = [
+	# spell:disable-line
+	"(?Rm)^.*(%|#|//)\\s*spell:disable-line$",
+
+	# spell:<on|off>
+	"(?s)(%|#|//)\\s*spell:off.*?\\n\\s*(%|#|//)\\s*spell:on",
+]