commit cd49935e338446a6ea6d4f29b67e269eef33eaab Author: Mark Date: Sat Oct 1 17:30:14 2022 -0700 Added jupyter docker files diff --git a/jupyter/README.md b/jupyter/README.md new file mode 100644 index 0000000..d18f9b4 --- /dev/null +++ b/jupyter/README.md @@ -0,0 +1,45 @@ +# Jupyter Docker Image + +This directory contains build files for a complete jupyter environment. It is based off the following repositories: + + - https://github.com/jupyter/docker-stacks + - https://github.com/rgriffogoes/scraper-notebook + - https://github.com/sharpTrick/sage-notebook + +The images these files produce contain everything you could possibly want in a jupyter notebook. Unfortunately, this also means that they take forever to build. + + +## Building the Image + +To build this image, run `build/build.fish.` Edit the script to customize the image. + + +## Image contents + - jupyterlab, jupyterhub, notebook + - latex + pandoc for exporting + - mamba for packages + - Python + - Scrape tools (bs4, requests) + - scipy, scikit-learn, scikit-image + - tensorflow + - sympy + - numpy + - matplotlib + - pandas + - Octave + - R + + +## TODO + - User config + - Volume permissions + - Auto dark theme + - Fix sage + - Selenium (broken) + - GPU support ([source](https://github.com/iot-salzburg/gpu-jupyter)) + - Haskell ([source](https://github.com/IHaskell/ihaskell-notebook)) + - Rust Kernel + - C++ Kernel + - Julia Kernel + - Perl Kernel + - Maxima Kernel diff --git a/jupyter/build/base/Dockerfile b/jupyter/build/base/Dockerfile new file mode 100644 index 0000000..d45f767 --- /dev/null +++ b/jupyter/build/base/Dockerfile @@ -0,0 +1,201 @@ +FROM debian:bullseye + +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +USER root + + +# Install all packages +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update --yes && \ + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + # Basics + # bzip2: to extract micromamba + # tini: see https://github.com/krallin/tini#why-tini + bzip2 \ + ca-certificates \ + fonts-liberation \ + locales \ + wget \ + sudo \ + tini \ + gcc \ + make \ + gnupg \ + curl \ + # Common tools + # less: needed to run help in R (https://github.com/jupyter/docker-stacks/issues/1588) + # ffmpeg: for matplotlib anim + # cm-super, dvipng: for latex labels + # build-essential: for cython + build-essential \ + cm-super \ + dvipng \ + git \ + nano \ + tzdata \ + unzip \ + vim \ + openssh-client \ + less \ + ffmpeg \ + # Jupyter dependencies + # pandoc: convert notebooks to html + # texlive-*: for nbconvert + pandoc \ + texlive \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-plain-generic \ + # Misc dependencies + # build-essential: for cython + # cm-super, dvipng: for latex labels + # ffmpeg: for matplotlib anim + build-essential \ + cm-super \ + dvipng \ + ffmpeg \ + && \ + # Clean up and generate locales + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH="${CONDA_DIR}/bin:${PATH}" \ + HOME="/home/${NB_USER}" + +# Copy a script that we will use to correct permissions after running certain commands +COPY fix-permissions.sh /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default NB_USER +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ + # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 + echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + +# Create user +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd \ + --no-log-init \ + --create-home \ + --no-user-group -u "${NB_UID}" \ + --shell /bin/bash \ + "${NB_USER}" \ + && \ + mkdir -p "${CONDA_DIR}" && \ + chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ + chmod g+w /etc/passwd && \ + fix-permissions "${HOME}" && \ + fix-permissions "${CONDA_DIR}" + +USER ${NB_UID} + + +# Pin python version here, or set it to "default" +ARG PYTHON_VERSION=3.10 + +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba, Jupyter Notebook, Lab, and Hub +# Generate a notebook server config +# Cleanup temporary files and remove Micromamba +# Correct permissions +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + # Get micromamba + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget -qO /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # + # Install packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + # Jupyter base + "mamba" \ + "notebook" \ + "jupyterhub" \ + "jupyterlab" \ + && \ + # Cleanup + rm micromamba && \ + # Pin major.minor version of python + mamba list python | \ + grep "^python " | \ + tr -s " " | \ + cut -d " " -f 1,2 \ + >> "${CONDA_DIR}/conda-meta/pinned" \ + && \ + jupyter notebook --generate-config && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + + +EXPOSE 8888 + +ENTRYPOINT ["tini", "-g", "--"] +CMD ["start-notebook.sh"] + +COPY start.sh start-notebook.sh start-singleuser.sh /usr/local/bin/ +COPY jupyter_server_config.py /etc/jupyter/ + +# Add R mimetype option to specify how the plot returns from R to the browser +COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ + +USER root +#COPY overrides.json /opt/conda/share/jupyter/lab/settings/ + +RUN mkdir "${HOME}/notebooks" && \ + chown "${NB_UID}:${NB_GID}" "${HOME}/notebooks" &&\ + chmod u+rwx "${HOME}/notebooks" +VOLUME "${HOME}/notebooks" + + +# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) +#RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ +# /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ +# fix-permissions /etc/jupyter/ + +# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands +# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 +HEALTHCHECK --interval=15s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -O- --no-verbose --tries=1 --no-check-certificate \ + http${GEN_CERT:+s}://localhost:8888${JUPYTERHUB_SERVICE_PREFIX:-/}api || exit 1 + +USER ${NB_UID} +WORKDIR "${HOME}" \ No newline at end of file diff --git a/jupyter/build/base/Rprofile.site b/jupyter/build/base/Rprofile.site new file mode 100644 index 0000000..3d6a93c --- /dev/null +++ b/jupyter/build/base/Rprofile.site @@ -0,0 +1,4 @@ +# Add R mimetype to specify how the plot returns from R to the browser. +# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter + +options(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf')) diff --git a/jupyter/build/base/fix-permissions.sh b/jupyter/build/base/fix-permissions.sh new file mode 100755 index 0000000..604900b --- /dev/null +++ b/jupyter/build/base/fix-permissions.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# set permissions on a directory +# after any installation, if a directory needs to be (human) user-writable, +# run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} +# and writable by that group. +# Deployments that want to set a specific user id can preserve permissions +# by adding the `--group-add users` line to `docker run`. + +# uses find to avoid touching files that already have the right permissions, +# which would cause massive image explosion + +# right permissions are: +# group=${NB_GID} +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "${d}" \ + ! \( \ + -group "${NB_GID}" \ + -a -perm -g+rwX \ + \) \ + -exec chgrp "${NB_GID}" -- {} \+ \ + -exec chmod g+rwX -- {} \+ + # setuid, setgid *on directories only* + find "${d}" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 -- {} \+ +done diff --git a/jupyter/build/base/initial-condarc b/jupyter/build/base/initial-condarc new file mode 100644 index 0000000..383aad3 --- /dev/null +++ b/jupyter/build/base/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge diff --git a/jupyter/build/base/jupyter_server_config.py b/jupyter/build/base/jupyter_server_config.py new file mode 100644 index 0000000..4c23d14 --- /dev/null +++ b/jupyter/build/base/jupyter_server_config.py @@ -0,0 +1,58 @@ +import os +import stat +import subprocess + +from jupyter_core.paths import jupyter_data_dir + +c = get_config() # noqa: F821 +c.ServerApp.ip = "0.0.0.0" +c.ServerApp.port = 8888 +c.ServerApp.open_browser = False +c.ServerApp.root_dir = "/home/joyvan/notebooks" + +# to output both image/svg+xml and application/pdf plot formats in the notebook file +c.InlineBackend.figure_formats = {"png", "jpeg", "svg", "pdf"} + +# https://github.com/jupyter/notebook/issues/3130 +c.FileContentsManager.delete_to_trash = False + +# Generate a self-signed certificate +OPENSSL_CONFIG = """\ +[req] +distinguished_name = req_distinguished_name +[req_distinguished_name] +""" +if "GEN_CERT" in os.environ: + dir_name = jupyter_data_dir() + pem_file = os.path.join(dir_name, "notebook.pem") + os.makedirs(dir_name, exist_ok=True) + + # Generate an openssl.cnf file to set the distinguished name + cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") + if not os.path.isfile(cnf_file): + with open(cnf_file, "w") as fh: + fh.write(OPENSSL_CONFIG) + + # Generate a certificate if one doesn't exist on disk + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-newkey=rsa:2048", + "-days=365", + "-nodes", + "-x509", + "-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated", + f"-keyout={pem_file}", + f"-out={pem_file}", + ] + ) + # Restrict access to the file + os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) + c.ServerApp.certfile = pem_file + +# Change default umask for all subprocesses of the notebook server if set in +# the environment +if "NB_UMASK" in os.environ: + os.umask(int(os.environ["NB_UMASK"], 8)) diff --git a/jupyter/build/base/overrides.json b/jupyter/build/base/overrides.json new file mode 100644 index 0000000..1ae6394 --- /dev/null +++ b/jupyter/build/base/overrides.json @@ -0,0 +1,5 @@ +{ + "@jupyterlab/apputils-extension:themes": { + "theme": "JupyterLab Dark" + } +} diff --git a/jupyter/build/base/start-notebook.sh b/jupyter/build/base/start-notebook.sh new file mode 100755 index 0000000..6c3be0a --- /dev/null +++ b/jupyter/build/base/start-notebook.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# The Jupyter command to launch +# JupyterLab by default +DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" + +if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then + echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." + exec /usr/local/bin/start-singleuser.sh "$@" +fi + +wrapper="" +if [[ "${RESTARTABLE}" == "yes" ]]; then + wrapper="" +fi + +exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" diff --git a/jupyter/build/base/start-singleuser.sh b/jupyter/build/base/start-singleuser.sh new file mode 100755 index 0000000..c040711 --- /dev/null +++ b/jupyter/build/base/start-singleuser.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# set default ip to 0.0.0.0 +if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then + NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" +fi + +# shellcheck disable=SC1091,SC2086 +. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" diff --git a/jupyter/build/base/start.sh b/jupyter/build/base/start.sh new file mode 100755 index 0000000..f78164f --- /dev/null +++ b/jupyter/build/base/start.sh @@ -0,0 +1,262 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# The _log function is used for everything this script wants to log. It will +# always log errors and warnings, but can be silenced for other messages +# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. +_log () { + if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then + echo "$@" + fi +} +_log "Entered start.sh with args:" "$@" + +# The run-hooks function looks for .sh scripts to source and executable files to +# run within a passed directory. +run-hooks () { + if [[ ! -d "${1}" ]] ; then + return + fi + _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" + for f in "${1}/"*; do + case "${f}" in + *.sh) + _log "${0}: running script ${f}" + # shellcheck disable=SC1090 + source "${f}" + ;; + *) + if [[ -x "${f}" ]] ; then + _log "${0}: running executable ${f}" + "${f}" + else + _log "${0}: ignoring non-executable ${f}" + fi + ;; + esac + done + _log "${0}: done running hooks in ${1}" +} + +# A helper function to unset env vars listed in the value of the env var +# JUPYTER_ENV_VARS_TO_UNSET. +unset_explicit_env_vars () { + if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then + for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do + echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" + unset "${env_var_to_unset}" + done + unset JUPYTER_ENV_VARS_TO_UNSET + fi +} + + +# Default to starting bash if no command was specified +if [ $# -eq 0 ]; then + cmd=( "bash" ) +else + cmd=( "$@" ) +fi + +# NOTE: This hook will run as the user the container was started with! +run-hooks /usr/local/bin/start-notebook.d + +# If the container started as the root user, then we have permission to refit +# the jovyan user, and ensure file permissions, grant sudo rights, and such +# things before we run the command passed to start.sh as the desired user +# (NB_USER). +# +if [ "$(id -u)" == 0 ] ; then + # Environment variables: + # - NB_USER: the desired username and associated home folder + # - NB_UID: the desired user id + # - NB_GID: a group id we want our user to belong to + # - NB_GROUP: a group name we want for the group + # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights + # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder + # - CHOWN_EXTRA: a comma separated list of paths to chown + # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands + + # Refit the jovyan user to the desired the user (NB_USER) + if id jovyan &> /dev/null ; then + if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then + _log "Updated the jovyan user:" + _log "- username: jovyan -> ${NB_USER}" + _log "- home dir: /home/jovyan -> /home/${NB_USER}" + fi + elif ! id -u "${NB_USER}" &> /dev/null; then + _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." + exit 1 + fi + # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is + # a member of the desired group (NB_GROUP, NB_GID) + if [ "${NB_UID}" != "$(id -u "${NB_USER}")" ] || [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then + _log "Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}" + # Ensure the desired group's existence + if [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then + groupadd --force --gid "${NB_GID}" --non-unique "${NB_GROUP:-${NB_USER}}" + fi + # Recreate the desired user as we want it + userdel "${NB_USER}" + useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" + fi + + # Move or symlink the jovyan home directory to the desired users home + # directory if it doesn't already exist, and update the current working + # directory to the new location if needed. + if [[ "${NB_USER}" != "jovyan" ]]; then + if [[ ! -e "/home/${NB_USER}" ]]; then + _log "Attempting to copy /home/jovyan to /home/${NB_USER}..." + mkdir "/home/${NB_USER}" + if cp -a /home/jovyan/. "/home/${NB_USER}/"; then + _log "Success!" + else + _log "Failed to copy data from /home/jovyan to /home/${NB_USER}!" + _log "Attempting to symlink /home/jovyan to /home/${NB_USER}..." + if ln -s /home/jovyan "/home/${NB_USER}"; then + _log "Success creating symlink!" + else + _log "ERROR: Failed copy data from /home/jovyan to /home/${NB_USER} or to create symlink!" + exit 1 + fi + fi + fi + # Ensure the current working directory is updated to the new path + if [[ "${PWD}/" == "/home/jovyan/"* ]]; then + new_wd="/home/${NB_USER}/${PWD:13}" + _log "Changing working directory to ${new_wd}" + cd "${new_wd}" + fi + fi + + # Optionally ensure the desired user get filesystem ownership of it's home + # folder and/or additional folders + if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then + _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" + # shellcheck disable=SC2086 + chown ${CHOWN_HOME_OPTS} "${NB_UID}:${NB_GID}" "/home/${NB_USER}" + fi + if [ -n "${CHOWN_EXTRA}" ]; then + for extra_dir in $(echo "${CHOWN_EXTRA}" | tr ',' ' '); do + _log "Ensuring ${extra_dir} is owned by ${NB_UID}:${NB_GID} ${CHOWN_EXTRA_OPTS:+(chown options: ${CHOWN_EXTRA_OPTS})}" + # shellcheck disable=SC2086 + chown ${CHOWN_EXTRA_OPTS} "${NB_UID}:${NB_GID}" "${extra_dir}" + done + fi + + # Update potentially outdated environment variables since image build + export XDG_CACHE_HOME="/home/${NB_USER}/.cache" + + # Prepend ${CONDA_DIR}/bin to sudo secure_path + sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path + + # Optionally grant passwordless sudo rights for the desired user + if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == "yes" ]]; then + _log "Granting ${NB_USER} passwordless sudo rights!" + echo "${NB_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/added-by-start-script + fi + + # NOTE: This hook is run as the root user! + run-hooks /usr/local/bin/before-notebook.d + + unset_explicit_env_vars + _log "Running as ${NB_USER}:" "${cmd[@]}" + exec sudo --preserve-env --set-home --user "${NB_USER}" \ + PATH="${PATH}" \ + PYTHONPATH="${PYTHONPATH:-}" \ + "${cmd[@]}" + # Notes on how we ensure that the environment that this container is started + # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when + # we transition from running as root to running as NB_USER. + # + # - We use `sudo` to execute the command as NB_USER. What then + # happens to the environment will be determined by configuration in + # /etc/sudoers and /etc/sudoers.d/* as well as flags we pass to the sudo + # command. The behavior can be inspected with `sudo -V` run as root. + # + # ref: `man sudo` https://linux.die.net/man/8/sudo + # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html + # + # - We use the `--preserve-env` flag to pass through most environment + # variables, but understand that exceptions are caused by the sudoers + # configuration: `env_delete` and `env_check`. + # + # - We use the `--set-home` flag to set the HOME variable appropriately. + # + # - To reduce the default list of variables deleted by sudo, we could have + # used `env_delete` from /etc/sudoers. It has higher priority than the + # `--preserve-env` flag and the `env_keep` configuration. + # + # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo + # resolves `${cmd[@]}` using the "secure_path" variable we modified + # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above + # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant + # for resolving paths of any subprocesses spawned by `${cmd[@]}`. + +# The container didn't start as the root user, so we will have to act as the +# user we started as. +else + # Warn about misconfiguration of: granting sudo rights + if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == "yes" ]]; then + _log "WARNING: container must be started as root to grant sudo permissions!" + fi + + JOVYAN_UID="$(id -u jovyan 2>/dev/null)" # The default UID for the jovyan user + JOVYAN_GID="$(id -g jovyan 2>/dev/null)" # The default GID for the jovyan user + + # Attempt to ensure the user uid we currently run as has a named entry in + # the /etc/passwd file, as it avoids software crashing on hard assumptions + # on such entry. Writing to the /etc/passwd was allowed for the root group + # from the Dockerfile during build. + # + # ref: https://github.com/jupyter/docker-stacks/issues/552 + if ! whoami &> /dev/null; then + _log "There is no entry in /etc/passwd for our UID=$(id -u). Attempting to fix..." + if [[ -w /etc/passwd ]]; then + _log "Renaming old jovyan user to nayvoj ($(id -u jovyan):$(id -g jovyan))" + + # We cannot use "sed --in-place" since sed tries to create a temp file in + # /etc/ and we may not have write access. Apply sed on our own temp file: + sed --expression="s/^jovyan:/nayvoj:/" /etc/passwd > /tmp/passwd + echo "${NB_USER}:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd + cat /tmp/passwd > /etc/passwd + rm /tmp/passwd + + _log "Added new ${NB_USER} user ($(id -u):$(id -g)). Fixed UID!" + + if [[ "${NB_USER}" != "jovyan" ]]; then + _log "WARNING: user is ${NB_USER} but home is /home/jovyan. You must run as root to rename the home directory!" + fi + else + _log "WARNING: unable to fix missing /etc/passwd entry because we don't have write permission. Try setting gid=0 with \"--user=$(id -u):0\"." + fi + fi + + # Warn about misconfiguration of: desired username, user id, or group id. + # A misconfiguration occurs when the user modifies the default values of + # NB_USER, NB_UID, or NB_GID, but we cannot update those values because we + # are not root. + if [[ "${NB_USER}" != "jovyan" && "${NB_USER}" != "$(id -un)" ]]; then + _log "WARNING: container must be started as root to change the desired user's name with NB_USER=\"${NB_USER}\"!" + fi + if [[ "${NB_UID}" != "${JOVYAN_UID}" && "${NB_UID}" != "$(id -u)" ]]; then + _log "WARNING: container must be started as root to change the desired user's id with NB_UID=\"${NB_UID}\"!" + fi + if [[ "${NB_GID}" != "${JOVYAN_GID}" && "${NB_GID}" != "$(id -g)" ]]; then + _log "WARNING: container must be started as root to change the desired user's group id with NB_GID=\"${NB_GID}\"!" + fi + + # Warn if the user isn't able to write files to ${HOME} + if [[ ! -w /home/jovyan ]]; then + _log "WARNING: no write access to /home/jovyan. Try starting the container with group 'users' (100), e.g. using \"--group-add=users\"." + fi + + # NOTE: This hook is run as the user we started the container as! + run-hooks /usr/local/bin/before-notebook.d + unset_explicit_env_vars + _log "Executing the command:" "${cmd[@]}" + exec "${cmd[@]}" +fi diff --git a/jupyter/build/build.fish b/jupyter/build/build.fish new file mode 100755 index 0000000..9e20362 --- /dev/null +++ b/jupyter/build/build.fish @@ -0,0 +1,61 @@ +#!/usr/bin/fish + +set img_idx 0 +set is_quiet true +set root_dir "." + +function build_image + # First argument: (optional) remove parent image? (true or false) + # Second argument: file name to build + + # Parse arguments + if test (count $argv) -eq 2 + set remove_previous $argv[1] + set build_file $argv[2] + else if test (count $argv) -eq 1 + set remove_previous true + set build_file $argv[1] + end + + printf "Building $build_file...\n" + + docker build $root_dir \ + --quiet=$is_quiet \ + --build-arg BASE_CONTAINER=betalupi/jupyter-inter-$img_idx \ + -f $build_file.Dockerfile \ + -t betalupi/jupyter-inter-(math $img_idx + 1) + + + if $remove_previous + printf "Done, cleaning up.\n\n" + sleep 1 + docker image rm betalupi/jupyter-inter-$img_idx + else + printf "Done. \n\n" + end + + set img_idx (math $img_idx + 1) +end + + + +# Base image MUST be built first +printf "Building base image...\n" +docker $root_dir/build \ + --quiet=$is_quiet \ + -t betalupi/jupyter-inter-0 +printf "Done. \n\n" + + + + +build_image false octave +build_image false r +build_image false sage + +build_image false plugins +build_image pymodules + +# Rename final image +docker image tag betalupi/jupyter-inter-(math $img_idx) betalupi/jupyter +docker image rm betalupi/jupyter-inter-(math $img_idx) \ No newline at end of file diff --git a/jupyter/build/octave.Dockerfile b/jupyter/build/octave.Dockerfile new file mode 100644 index 0000000..2e75353 --- /dev/null +++ b/jupyter/build/octave.Dockerfile @@ -0,0 +1,24 @@ +ARG BASE_CONTAINER=betalupi/jupyter-base +FROM $BASE_CONTAINER +LABEL maintainer="Mark " + +USER root +RUN apt-get update --yes && apt-get install --yes --no-install-recommends \ + octave \ + gnuplot \ + && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} +RUN mamba install --quiet --yes \ + "octave_kernel" \ + && \ + # Cleanup + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +USER ${NB_UID} diff --git a/jupyter/build/plugins.Dockerfile b/jupyter/build/plugins.Dockerfile new file mode 100644 index 0000000..c8f8045 --- /dev/null +++ b/jupyter/build/plugins.Dockerfile @@ -0,0 +1,16 @@ +ARG BASE_CONTAINER=betalupi/jupyter-base +FROM $BASE_CONTAINER +LABEL maintainer="Mark " + +USER ${NB_UID} + +RUN mamba install --yes \ + "jupyterlab-git" \ + && \ + # Cleanup + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" \ No newline at end of file diff --git a/jupyter/build/pymodules.Dockerfile b/jupyter/build/pymodules.Dockerfile new file mode 100644 index 0000000..698d5e0 --- /dev/null +++ b/jupyter/build/pymodules.Dockerfile @@ -0,0 +1,82 @@ +ARG BASE_CONTAINER=betalupi/jupyter-base +FROM $BASE_CONTAINER +LABEL maintainer="Mark " + +USER ${NB_UID} + +RUN mamba install --yes \ + # Python packages + # Scraping + "beautifulsoup4" \ + "requests" \ + "schedule" \ + # Misc + "altair" \ + "bokeh" \ + "bottleneck" \ + "cloudpickle" \ + "conda-forge::blas=*=openblas" \ + "cython" \ + "dask" \ + "dill" \ + "h5py" \ + "ipympl" \ + "ipywidgets" \ + "matplotlib-base" \ + "numba" \ + "numpy" \ + "numexpr" \ + "pandas" \ + "patsy" \ + "protobuf" \ + "pytables" \ + "scikit-image" \ + "scikit-learn" \ + "scipy" \ + "seaborn" \ + "sqlalchemy" \ + "statsmodels" \ + "sympy" \ + "widgetsnbextension" \ + "xlrd" \ + "tensorflow" \ + && \ + # Cleanup + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Import matplotlib the first time to build the font cache. +ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" +RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ + fix-permissions "/home/${NB_USER}" + + + +#FROM pymodules as selenium +#USER ${NB_UID} +# +#RUN mamba install --yes \ +# "selenium" \ +# && \ +# # Cleanup +# mamba clean --all -f -y && \ +# npm cache clean --force && \ +# jupyter lab clean && \ +# rm -rf "/home/${NB_USER}/.cache/yarn" && \ +# fix-permissions "${CONDA_DIR}" && \ +# fix-permissions "/home/${NB_USER}" +# +## Install google chrome +#RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +#RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/#sources.list.d/google-chrome.list' +#RUN apt-get -y update +#RUN apt-get install -y google-chrome-stable +# +## Install chromedriver +#RUN apt-get install -yqq unzip +#RUN wget -O /tmp/chromedriver.zip http://chromedriver.storage.googleapis.com/`curl -sS #chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip +#RUN unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/ \ No newline at end of file diff --git a/jupyter/build/r.Dockerfile b/jupyter/build/r.Dockerfile new file mode 100644 index 0000000..b11aa94 --- /dev/null +++ b/jupyter/build/r.Dockerfile @@ -0,0 +1,57 @@ +ARG BASE_CONTAINER=betalupi/jupyter-base +FROM $BASE_CONTAINER +LABEL maintainer="Mark " + +# R pre-requisites +USER root +RUN apt-get update --yes && apt-get install --yes --no-install-recommends \ + fonts-dejavu \ + unixodbc \ + unixodbc-dev \ + r-cran-rodbc \ + gfortran \ + gcc && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +USER ${NB_UID} +RUN mamba install --quiet --yes \ + "r-base" \ + "r-caret" \ + "r-crayon" \ + "r-devtools" \ + "r-e1071" \ + "r-forecast" \ + "r-hexbin" \ + "r-htmltools" \ + "r-htmlwidgets" \ + "r-irkernel" \ + "r-nycflights13" \ + "r-randomforest" \ + "r-rcurl" \ + "r-rmarkdown" \ + "r-rodbc" \ + "r-rsqlite" \ + "r-shiny" \ + "r-tidyverse" \ + "unixodbc" && \ + # Cleanup + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# `rpy2` and `r-tidymodels` are not easy to install under arm +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" == "x86_64" ]; then \ + mamba install --quiet --yes \ + "rpy2" \ + "r-tidymodels" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}"; \ + fi; + +USER ${NB_UID} \ No newline at end of file diff --git a/jupyter/docker-compose.yml b/jupyter/docker-compose.yml new file mode 100644 index 0000000..26522b3 --- /dev/null +++ b/jupyter/docker-compose.yml @@ -0,0 +1,14 @@ +version: "3" + +services: + jupyter: + image: git.betalupi.com/mark/jupyter + container_name: jupyter + restart: unless-stopped + + volumes: + - "./notebooks:/home/joyvan/notebooks" + + # host:container + ports: + - "8888:8888"