From bc8fedb54a2724cd77d976dae0e3023f05682bc8 Mon Sep 17 00:00:00 2001 From: Mark Date: Sat, 25 Feb 2023 22:59:00 -0800 Subject: [PATCH] Squashed commit of jenkins changes --- .gitignore | 3 + resources/scripts/Jenkinsfile | 28 ++++++ resources/scripts/build.py | 149 +++++++++++++++++++++++++++++++ resources/scripts/build.toml | 12 +++ resources/scripts/helpers/tex.py | 60 +++++++++++++ 5 files changed, 252 insertions(+) create mode 100644 resources/scripts/Jenkinsfile create mode 100644 resources/scripts/build.py create mode 100644 resources/scripts/build.toml create mode 100644 resources/scripts/helpers/tex.py diff --git a/.gitignore b/.gitignore index 473f474..474eddc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +venv +__pycache__ + # Output files main.pdf diff --git a/resources/scripts/Jenkinsfile b/resources/scripts/Jenkinsfile new file mode 100644 index 0000000..26b2f20 --- /dev/null +++ b/resources/scripts/Jenkinsfile @@ -0,0 +1,28 @@ +pipeline { + agent none + environment { + NC_CRED = credentials("nc-jenkins-cred") + NO_TEST = true + } + stages { + stage("Build") { + agent { + docker { + image "git.betalupi.com/mark/latex:latest" + } + } + steps { + // Keep these in one "sh" directive. + // Todo: create setup files so we don't + // have to pip install manually here. + // Maybe integrate python packages into the docker container? + sh ''' + python -m venv venv + source venv/bin/activate + pip install tomli webdavclient3 + python resources/scripts/build.py + ''' + } + } + } +} diff --git a/resources/scripts/build.py b/resources/scripts/build.py new file mode 100644 index 0000000..12826bf --- /dev/null +++ b/resources/scripts/build.py @@ -0,0 +1,149 @@ +# Build script for CI. +# This file is run from the root of the repository + +import os +import tomli +from pathlib import Path +import shutil + +from webdav3.client import Client + +import helpers.tex as tex + +conf_path = "resources/scripts/build.toml" + +# Prepare envvars +e = os.environ.copy() +e["TEXINPUTS"] = f"::/usr/share/texmf-dist/tex//" +print(Path.cwd()) + +# If true, rebuild everything +# without checking for changes. +if "FORCE_ALL" in e: + force_all = e["FORCE_ALL"] == "true" +else: + force_all = False + +if "NO_TEST" in e: + test = e["NO_TEST"] == "false" +else: + test = True + + +# Used only for debug +class FakeClient: + def __init__(self): + pass + + def mkdir(self, path): + print(f"Making dir {path}") + + def upload_sync(self, local_path, remote_path): + print(f"Sync {local_path} to {remote_path}") + + def clean(self, path): + print(f"Cleaning {path}") + + + +# Load configuration +with open(conf_path, mode="rb") as fp: + config = tomli.load(fp) + +# Prepare directories +builddir = Path().absolute() / config["core"]["build_dir"] +output = Path().absolute() / config["core"]["output_dir"] +output.mkdir(parents = True, exist_ok = True) +builddir.mkdir(parents = True, exist_ok = True) + + + +if not test: + # Connect to webdav + client = Client({ + "webdav_login": e["NC_CRED_USR"], + "webdav_password": e["NC_CRED_PSW"], + "webdav_hostname": e["DAV_HOSTNAME"], # https://host:port + "webdav_root": e["DAV_ROOT"] # the rest of the url + # client.list() will break if we do not seperate the hostname + # and the root path. + }) +else: + client = FakeClient() + + +def upload_dir(local_path: Path): + for p in local_path.rglob("*"): + r = p.relative_to(output) + if p.is_dir(): + client.mkdir(str(r)) + + elif p.is_file(): + client.upload_sync( # type: ignore + remote_path = str(r), + local_path = p + ) + + +# Print envvars for debug +#print(e) + + +for i, item in enumerate(config["dir-of-dirs"]): + p = Path(item["path"]) + + for s in p.iterdir(): + # s: path to directory, + # # relative to repo root + if not s.is_dir(): + continue + + print(f"Building {s}/main.tex") + + # Make pdf + pdf_nosolutions = tex.build( + s, + mainfile = "main.tex", + jobname = f"{i}-nosols", + build_dir = builddir, + solutions = False, + + env = e, + test = test + ) + + pdf_solutions = tex.build( + s, + mainfile = "main.tex", + jobname = f"{i}-sols", + build_dir = builddir, + solutions = True, + + env = e, + test = test + ) + + target_dir = output / s.parent + target_dir.mkdir(parents = True, exist_ok = True) + + shutil.copy( + pdf_nosolutions, + target_dir / (s.stem + ".pdf") + ) + + shutil.copy( + pdf_solutions, + target_dir / (s.stem + ".sols.pdf") + ) + +if not test: + # Delete old files + for i in client.list(): + if i.endswith("/"): + i = i[:-1] + client.clean(i) + +upload_dir(output) + +# Remove output files so they don't persist +shutil.rmtree(output) \ No newline at end of file diff --git a/resources/scripts/build.toml b/resources/scripts/build.toml new file mode 100644 index 0000000..ef5c3fb --- /dev/null +++ b/resources/scripts/build.toml @@ -0,0 +1,12 @@ +# All paths are relative to the root of this repository. + +[core] +build_dir = "_build" +output_dir = "_output" + + +[[dir-of-dirs]] +path = "Advanced" + +[[dir-of-dirs]] +path = "Intermediate" \ No newline at end of file diff --git a/resources/scripts/helpers/tex.py b/resources/scripts/helpers/tex.py new file mode 100644 index 0000000..8350e6b --- /dev/null +++ b/resources/scripts/helpers/tex.py @@ -0,0 +1,60 @@ +from pathlib import Path +import subprocess + +def build( + path: Path, + *, + solutions: bool = True, + mainfile: str = "main.tex", + jobname: str = "main", + build_dir: Path = Path("build"), + + test = False, + env: dict = {}, + ) -> Path: + """ + Build a directory using XeTeX. + + Args: + path (Path): directory to build + solutions (bool, optional): Create a handout with solutions? Defaults to True. + mainfile (str, optional): Path to main TeX file, relative to path. Defaults to "main.tex". + jobname (str, optional): TeX job name. Defaults to "main". + build_dir (Path, optional): Where to place build files. Defaults to Path("build"). + + test (bool, optional): If true, dry run. Defaults to False. + env (dict): Dictionary of environment variables. + + Returns: + Path: Absolute path to output pdf. + """ + + # Make temporary file for TeX build arg hack + if solutions: + with (build_dir/"tmp.tex").open("w") as f: + f.write("\\def\\argYesSolutions{1}\\input{" + mainfile + "}") + else: + with (build_dir/"tmp.tex").open("w") as f: + f.write("\\def\\argNoSolutions{1}\\input{" + mainfile + "}") + + cmd = subprocess.run([ + "latexmk", + "-interaction=nonstopmode", + "-file-line-error", + f"-outdir={build_dir}", + "-xelatex", + f"-jobname={jobname}", + build_dir/"tmp.tex" + ], + cwd = path, + env = env, + stdout = None if test else subprocess.DEVNULL, + stderr = None if test else subprocess.DEVNULL + ) + + assert cmd.returncode == 0 + + # Remove tmp file + (build_dir / "tmp.tex").unlink() + + return build_dir / f"{jobname}.pdf"