diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..615b9a6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + typos: + name: "Typos" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check typos + uses: crate-ci/typos@master + with: + config: ./tools/typos.toml + + typstyle: + name: "Typst formatting" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: "Download Typstyle" + run: | + wget -q "https://github.com/Enter-tainer/typstyle/releases/download/v0.12.14/typstyle-x86_64-unknown-linux-musl" + chmod +x typstyle-x86_64-unknown-linux-musl + + - name: Check typst formatting + run: | + find . -name "*.typ" -type f -print0 | xargs -0 \ + ./typstyle-x86_64-unknown-linux-musl --check + + build: + needs: + - typos + - typstyle + + name: "Build" + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + # Could instead install `texlive-full`, but that takes ~20 minutes. + # We'll specify the packages we need manually. + - name: "Install TeXLive" + run: | + sudo apt update + DEBIAN_FRONTEND=noninteractive \ + sudo apt install --yes \ + texlive texlive-xetex \ + texlive-games texlive-fonts-extra texlive-latex-extra \ + texlive-pictures texlive-pstricks \ + python3-requests + + # Typst isn't packaged, and manual download gives us + # more control anyway. + - name: "Download Typst" + run: | + wget -q "https://github.com/typst/typst/releases/download/v0.12.0/typst-x86_64-unknown-linux-musl.tar.xz" + tar -xf "typst-x86_64-unknown-linux-musl.tar.xz" + mv "typst-x86_64-unknown-linux-musl/typst" . + rm "typst-x86_64-unknown-linux-musl.tar.xz" + rm -dr "typst-x86_64-unknown-linux-musl" + + # Builds all handouts, LaTeX and Typst + - name: "Build handouts" + run: TYPST_PATH="$(pwd)/typst" python tools/scripts/build.py + + # Upload logs, even if build fails. + # LaTeX stdout/stderr isn't always helpful. + - name: "Save LaTeX logs" + uses: actions/upload-artifact@v3 + if: always() + with: + name: "LaTeX logs" + path: "**/*.log" + retention-days: 14 + + # Upload build output + - name: "Save output" + uses: actions/upload-artifact@v3 + with: + name: "Build output" + path: "output/*" + retention-days: 7 + + - name: "Publish package (hash)" + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + PUBLISH_USER="${{ secrets.PUBLISH_USER }}" \ + PUBLISH_KEY="${{ secrets.PUBLISH_KEY }}" \ + VERSION="${{ github.sha }}" \ + PACKAGE="${{ vars.PACKAGE }}" \ + python tools/scripts/publish.py + + - name: "Publish package (latest)" + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + PUBLISH_USER="${{ secrets.PUBLISH_USER }}" \ + PUBLISH_KEY="${{ secrets.PUBLISH_KEY }}" \ + VERSION="latest" \ + PACKAGE="${{ vars.PACKAGE }}" \ + python tools/scripts/publish.py diff --git a/tools/meta.toml b/tools/meta.toml new file mode 100644 index 0000000..8147e05 --- /dev/null +++ b/tools/meta.toml @@ -0,0 +1,23 @@ +# This is a sample `meta.toml`. +# A copy of this file should exist in every handout directory. +# +# All keys are required. + +[metadata] +title = "title of this handout" + + +[publish] +# Should we publish this handout? +# If `false`, no part of this handout is published. +# (useful for drafts) +handout = true + +# Should we publish an "instructor's" version of this handout? +# This has no effect if `publish.handout == false`. +# +# If `true`, publish a version of this handout with solutions. +# If `false`, do not publish a handout with solutions. +# +# Set this to `false` if solutions have not been written. +solutions = true diff --git a/tools/scripts/build.py b/tools/scripts/build.py new file mode 100644 index 0000000..5dcc981 --- /dev/null +++ b/tools/scripts/build.py @@ -0,0 +1,302 @@ +from typing import TypedDict +from pathlib import Path +import subprocess +import tomllib +import shutil +import json +import os + +# TODO: +# list handouts without solutions +# list handouts that are not published +# lint: always commit without `nosolutions` +# use latexmk + +ROOT: Path = Path(os.getcwd()) + +### CONFIGURATION +OUT_DIR: Path = ROOT / "output" +TYPST_PATH: str = "typst" +XETEX_PATH: str = "xelatex" +### END CONFIGURATION + +# Allow path override +_env = os.environ.get("TYPST_PATH") +if isinstance(_env, str): + TYPST_PATH = _env + + +def log(msg): + print(f"[BUILD.PY] {msg}") + + +log(f"Running in {ROOT}") +if not ROOT.is_dir(): + log("Root is not a directory, cannot continue") + exit(1) + +log(f"Output dir is {OUT_DIR}") +if OUT_DIR.exists(): + log("Output dir exists, removing") + shutil.rmtree(OUT_DIR) + +OUT_DIR.mkdir(parents=True) + + +IndexEntry = TypedDict( + "IndexEntry", + {"title": str, "group": str, "handout_file": str, "solutions_file": str | None}, +) + + +MetaToml = TypedDict( + "MetaToml", {"title": str, "publish_handout": bool, "publish_solutions": bool} +) + + +def sanitize_file_name(file: str) -> str: + return file.replace("?", "") + + +def read_meta_toml(file: Path) -> MetaToml: + with file.open("rb") as f: + base = tomllib.load(f) + + meta = base.get("metadata") + if not isinstance(meta, dict): + log("Invalid meta.toml: `metadata` should be a table") + exit(1) + + title = meta.get("title") + if not isinstance(title, str): + log("Invalid meta.toml: `title` should be a string") + exit(1) + + pub = base.get("publish") + if not isinstance(pub, dict): + log("Invalid meta.toml: `publish` should be a table") + exit(1) + + pub_handout = pub.get("handout") + if not isinstance(pub_handout, bool): + log("Invalid meta.toml: `publish.handout` should be a boolean") + exit(1) + + pub_solutions = pub.get("solutions") + if not isinstance(pub_solutions, bool): + log("Invalid meta.toml: `publish.solutions` should be a boolean") + exit(1) + + return { + "title": title, + "publish_handout": pub_handout, + "publish_solutions": pub_solutions, + } + + +def log_error(res): + stdout = res.stdout.decode("utf-8").strip() + stderr = res.stderr.decode("utf-8").strip() + + log(f"Build failed with code {res.returncode}") + + if stdout == "": + log("Stdout: ") + else: + log("Stdout:") + print(stdout) + + if stderr == "": + log("Stderr: ") + else: + log("Stderr:") + print(stderr) + + exit(1) + + +def build_typst(source_dir: Path, out_subdir: Path) -> IndexEntry | None: + if not (source_dir / "main.typ").is_file(): + # log(f"No main.typ, skipping {source_dir}") + return None + + meta_path = source_dir / "meta.toml" + if not meta_path.is_file(): + log(f"No meta.toml, skipping {source_dir}") + return None + meta = read_meta_toml(meta_path) + + handout_file = sanitize_file_name(f"{meta['title']}.pdf") + solutions_file = sanitize_file_name(f"{meta['title']}.sols.pdf") + + # Do nothing if not published + if not meta["publish_handout"]: + return None + + # Build handout + log(f"Building typst (handout) : {source_dir}") + + out = OUT_DIR / out_subdir + out.mkdir(parents=True, exist_ok=True) + + res = subprocess.run( + [ + TYPST_PATH, + "compile", + "--ignore-system-fonts", + "main.typ", + "--input", + "show_solutions=false", + f"{out}/{handout_file}", + ], + cwd=source_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if res.returncode != 0: + log_error(res) + + # Build solutions + if meta["publish_solutions"]: + log(f"Building typst (solutions): {source_dir}") + res = subprocess.run( + [ + TYPST_PATH, + "compile", + "--ignore-system-fonts", + "main.typ", + f"{out}/{solutions_file}", + ], + cwd=source_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if res.returncode != 0: + log_error(res) + + return { + "title": meta["title"], + "group": str(out_subdir), + "handout_file": str(out / handout_file), + "solutions_file": ( + str(out / solutions_file) if meta["publish_solutions"] else None + ), + } + + +def build_xetex(source_dir: Path, out_subdir: Path) -> IndexEntry | None: + if not (source_dir / "main.tex").is_file(): + # log(f"No main.tex, skipping {source_dir}") + return None + + meta_path = source_dir / "meta.toml" + if not meta_path.is_file(): + log(f"No meta.toml, skipping {source_dir}") + return None + meta = read_meta_toml(meta_path) + + handout_file = sanitize_file_name(f"{meta['title']}.pdf") + solutions_file = sanitize_file_name(f"{meta['title']}.sols.pdf") + + # Do nothing if not published + if not meta["publish_handout"]: + return None + + # Build handout + log(f"Building xetex (handout) : {source_dir}") + + out = OUT_DIR / out_subdir + out.mkdir(parents=True, exist_ok=True) + + res = subprocess.run( + [ + XETEX_PATH, + "-interaction=batchmode", + ], + input=b"\\def\\argNoSolutions{1}\\input{main.tex}", + cwd=source_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + shutil.copy(source_dir / "main.pdf", f"{out}/{handout_file}") + except Exception as e: + log(f"Error: {e}") + log_error(res) + + if res.returncode != 0: + log_error(res) + + # Build solutions + if meta["publish_solutions"]: + log(f"Building xetex (solutions): {source_dir}") + res = subprocess.run( + [ + XETEX_PATH, + "-interaction=batchmode", + ], + input=b"\\def\\argYesSolutions{1}\\input{main.tex}", + cwd=source_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + shutil.copy(source_dir / "main.pdf", f"{out}/{solutions_file}") + except Exception as e: + log(f"Error: {e}") + log_error(res) + + if res.returncode != 0: + log_error(res) + + return { + "title": meta["title"], + "group": str(out_subdir), + "handout_file": str(out / handout_file), + "solutions_file": ( + str(out / solutions_file) if meta["publish_solutions"] else None + ), + } + + +# Try to build handouts in all subdirs of `base`, +# skipping those that do not contain a handout. +# +# This method does _not_ recurse into subdirectories. +def build_dir(base: str, out_sub: str, index: list[IndexEntry]): + builders = [build_typst, build_xetex] + + for d_str in os.listdir(base): + d = base / Path(d_str) + if not d.is_dir(): + continue + + # Try each builder, + # stopping at the first success + done = False + for builder in builders: + res = builder(d, Path(out_sub)) + if res is not None: + # Check for duplicate titles + for i in index: + if i["title"] == res["title"]: + log(f'Duplicate title "{i["title"]}", cannot continue') + exit(1) + # Save to index and exit + index.append(res) + done = True + if done: + break + + +index: list[IndexEntry] = [] +build_dir("src/Advanced", "Advanced", index) +build_dir("src/Intermediate", "Intermediate", index) +build_dir("src/Warm-Ups", "Warm-Ups", index) + +with open(OUT_DIR / "index.json", "w") as f: + f.write(json.dumps(index)) diff --git a/tools/scripts/publish.py b/tools/scripts/publish.py new file mode 100644 index 0000000..e102a42 --- /dev/null +++ b/tools/scripts/publish.py @@ -0,0 +1,95 @@ +# Publish the output of `build.py` +# as a Gitea package. + +from pathlib import Path +import requests +import json +import os +import re + +URL = "https://git.betalupi.com" +USER = os.environ["PUBLISH_USER"] +PACKAGE = os.environ["PACKAGE"] +VERSION = os.environ["VERSION"] +AUTH = requests.auth.HTTPBasicAuth(USER, os.environ["PUBLISH_KEY"]) + +ROOT: Path = Path(os.getcwd()) +SRC_DIR: Path = ROOT / "output" + + +def log(msg): + print(f"[PUBLISH.PY] {msg}") + + +log(f"Version is {VERSION}") +log(f"Package is {PACKAGE}") +log(f"Running in {ROOT}") +if not ROOT.is_dir(): + log("Root is not a directory, cannot continue") + exit(1) + +log(f"Source dir is {SRC_DIR}") +if not SRC_DIR.exists(): + log("Source dir doesn't exist, cannot continue") + exit(1) + + +def del_package(): + log(f"Deleting package {PACKAGE}/{VERSION}") + res = requests.delete( + f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}", + auth=AUTH, + ) + if res.status_code != 204 and res.status_code != 404: + log(f"Deletion failed with code {res.status_code}") + + +# Delete if already exists +# (important for the `latest` package) +del_package() + + +def upload(data, target: str): + target = re.sub("[^A-Za-z0-9_. -]+", "", target) + + res = requests.put( + f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}/{target}", + auth=AUTH, + data=data, + ) + + if res.status_code != 201: + log(f"Upload failed with code {res.status_code}") + del_package() # Do not keep partial package if upload fails + exit(1) + + return f"{URL}/api/packages/{USER}/generic/{PACKAGE}/{VERSION}/{target}" + + +index_file = SRC_DIR / "index.json" +with index_file.open("r") as f: + index = json.load(f) + +new_index = [] +for handout in index: + title = handout["title"] + group = handout["group"] + h_file = SRC_DIR / handout["handout_file"] + s_file = handout["solutions_file"] + if s_file is not None: + s_file = SRC_DIR / s_file + log(f"Uploading {title}") + + h_url = None + s_url = None + + h_url = upload(h_file.open("rb").read(), f"{group} - {title}.pdf") + if s_file is not None: + log(f"Uploading {title} solutions") + s_url = upload(s_file.open("rb").read(), f"{group} - {title}.sols.pdf") + + new_index.append( + {"title": title, "group": group, "handout": h_url, "solutions": s_url} + ) + +upload(json.dumps(new_index), "index.json") diff --git a/tools/scripts/ruff.toml b/tools/scripts/ruff.toml new file mode 100644 index 0000000..c8f72c4 --- /dev/null +++ b/tools/scripts/ruff.toml @@ -0,0 +1,18 @@ +exclude = ["venv"] +line-length = 88 +indent-width = 4 +target-version = "py39" +include = ["scripts/**/*.py"] + +[lint] +select = ["E4", "E7", "E9", "F"] +ignore = [] +fixable = ["ALL"] +unfixable = [] +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +quote-style = "double" +indent-style = "tab" +skip-magic-trailing-comma = false +line-ending = "lf" diff --git a/tools/typos.toml b/tools/typos.toml new file mode 100644 index 0000000..1688011 --- /dev/null +++ b/tools/typos.toml @@ -0,0 +1,11 @@ +[default] +extend-words."LSAT" = "LSAT" +extend-words."ket" = "ket" + +extend-ignore-re = [ + # spell:disable-line + "(?Rm)^.*(%|#|//)\\s*spell:disable-line$", + + # spell: + "(?s)(%|#|//)\\s*spell:off.*?\\n\\s*(%|#|//)\\s*spell:on", +]