From 87f2e77449b7741be542e61092fdb888476fd039 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Tue, 12 Nov 2019 22:22:20 +0100 Subject: [PATCH] Initial commit, local mode runs --- .gitignore | 5 + add-to-swarm.sh | 20 ++ docker-compose-swarm.yml | 36 +++ docker-compose.yml | 28 ++ remove-from-swarm.sh | 4 + src/Dockerfile | 424 +++++++++++++++++++++++++++++++ src/fix-permissions | 35 +++ src/jupyter_notebook_config.json | 6 + src/jupyter_notebook_config.py | 55 ++++ src/start-notebook.sh | 15 ++ src/start-singleuser.sh | 44 ++++ src/start.sh | 152 +++++++++++ start-local.sh | 4 + stop-local.sh | 4 + 14 files changed, 832 insertions(+) create mode 100755 add-to-swarm.sh create mode 100755 docker-compose-swarm.yml create mode 100755 docker-compose.yml create mode 100755 remove-from-swarm.sh create mode 100644 src/Dockerfile create mode 100755 src/fix-permissions create mode 100644 src/jupyter_notebook_config.json create mode 100644 src/jupyter_notebook_config.py create mode 100755 src/start-notebook.sh create mode 100755 src/start-singleuser.sh create mode 100755 src/start.sh create mode 100755 start-local.sh create mode 100755 stop-local.sh diff --git a/.gitignore b/.gitignore index 894a44c..d49c185 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,8 @@ venv.bak/ # mypy .mypy_cache/ + + +# individual stuff +/data +*.save* diff --git a/add-to-swarm.sh b/add-to-swarm.sh new file mode 100755 index 0000000..586a91f --- /dev/null +++ b/add-to-swarm.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +if [[ -n "$1" ]]; then + result=$(docker network ls) + if [[ "$result" == *" $1 "* ]]; then + echo "Adding gpu-jupyter to the swarm in the network $1." + export JUPYTER_NETWORK=$1 + # echo $JUPYTER_NETWORK + docker-compose -f docker-compose-swarm.yml build + docker-compose -f docker-compose-swarm.yml push +# docker stack deploy --compose-file docker-compose-swarm.yml gpu + else + echo "Could not find network $1. Please provide a valid docker network." + fi +else + echo "Usage: $0 network" + echo "Please set the network" +fi + diff --git a/docker-compose-swarm.yml b/docker-compose-swarm.yml new file mode 100755 index 0000000..c7a84a1 --- /dev/null +++ b/docker-compose-swarm.yml @@ -0,0 +1,36 @@ +version: "3.4" +services: + gpu-jupyter: + image: 127.0.0.1:5001/gpu-jupyter + build: ./src/ + ports: + - 8848:8888 + volumes: + - ./data:/home/jovyan/work + environment: + GRANT_SUDO: "yes" + JUPYTER_ENABLE_LAB: "yes" + # enable sudo permissions + user: + "root" + networks: + - default + - $JUPYTER_NETWORK + deploy: + placement: + constraints: [node.hostname == data-science] + resources: + limits: + cpus: "4" + memory: 8G + replicas: 1 + update_config: + parallelism: 2 + delay: 10s + restart_policy: + condition: on-failure + +networks: + $JUPYTER_NETWORK: + external: + name: $JUPYTER_NETWORK diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100755 index 0000000..f26e992 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.4" +services: + gpu-jupyter: + image: 127.0.0.1:5001/gpu-jupyter + build: ./src/ + ports: + - 8848:8888 + volumes: + - ./data:/home/jovyan/work + environment: + GRANT_SUDO: "yes" + JUPYTER_ENABLE_LAB: "yes" + # enable sudo permissions + user: + "root" + deploy: + placement: + constraints: [node.hostname == data-science] + resources: + limits: + cpus: "4" + memory: 8G + replicas: 1 + update_config: + parallelism: 2 + delay: 10s + restart_policy: + condition: on-failure diff --git a/remove-from-swarm.sh b/remove-from-swarm.sh new file mode 100755 index 0000000..e17cb64 --- /dev/null +++ b/remove-from-swarm.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +docker stack rm gpu_gpu-jupyter diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..b0acda4 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,424 @@ +FROM nvidia/cuda:10.1-base-ubuntu18.04 +LABEL maintainer="Christoph Schranz " + +############################################################################ +#################### Dependency: jupyter/base-image ######################## +############################################################################ + +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +USER root + +# Install all OS dependencies for notebook server that starts but lacks all +# features (e.g., download as all possible file formats) +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get -yq dist-upgrade \ + && apt-get install -yq --no-install-recommends \ + wget \ + bzip2 \ + ca-certificates \ + sudo \ + locales \ + fonts-liberation \ + && rm -rf /var/lib/apt/lists/* + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + NB_USER=$NB_USER \ + NB_UID=$NB_UID \ + NB_GID=$NB_GID \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH=$CONDA_DIR/bin:$PATH \ + HOME=/home/$NB_USER + +# Add a script that we will use to correct permissions after running certain commands +ADD fix-permissions /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default NB_USER +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc + +# Create NB_USER wtih name jovyan user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ + mkdir -p $CONDA_DIR && \ + chown $NB_USER:$NB_GID $CONDA_DIR && \ + chmod g+w /etc/passwd && \ + fix-permissions $HOME && \ + fix-permissions "$(dirname $CONDA_DIR)" + +USER $NB_UID +WORKDIR $HOME + +# Setup work directory for backward-compatibility +RUN mkdir /home/$NB_USER/work && \ + fix-permissions /home/$NB_USER + +# Install conda as jovyan and check the md5 sum provided on the download site +ENV MINICONDA_VERSION=4.6.14 \ + CONDA_VERSION=4.7.10 + +RUN cd /tmp && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ + echo "718259965f234088d785cad1fbd7de03 *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ + /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ + rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ + echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ + $CONDA_DIR/bin/conda config --system --prepend channels conda-forge && \ + $CONDA_DIR/bin/conda config --system --set auto_update_conda false && \ + $CONDA_DIR/bin/conda config --system --set show_channel_urls true && \ + $CONDA_DIR/bin/conda install --quiet --yes conda && \ + $CONDA_DIR/bin/conda update --all --quiet --yes && \ + conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \ + conda clean --all -f -y && \ + rm -rf /home/$NB_USER/.cache/yarn && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Install Tini +RUN conda install --quiet --yes 'tini=0.18.0' && \ + conda list tini | grep tini | tr -s ' ' | cut -d ' ' -f 1,2 >> $CONDA_DIR/conda-meta/pinned && \ + conda clean --all -f -y && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Install Jupyter Notebook, Lab, and Hub +# Generate a notebook server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +RUN conda install --quiet --yes \ + 'notebook=6.0.0' \ + 'jupyterhub=1.0.0' \ + 'jupyterlab=1.0.4' && \ + conda clean --all -f -y && \ + npm cache clean --force && \ + jupyter notebook --generate-config && \ + rm -rf $CONDA_DIR/share/jupyter/lab/staging && \ + rm -rf /home/$NB_USER/.cache/yarn && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +EXPOSE 8888 + +# Configure container startup +ENTRYPOINT ["tini", "-g", "--"] +CMD ["start-notebook.sh"] + +# Add local files as late as possible to avoid cache busting +COPY start.sh /usr/local/bin/ +COPY start-notebook.sh /usr/local/bin/ +COPY start-singleuser.sh /usr/local/bin/ +COPY jupyter_notebook_config.py /etc/jupyter/ + +# Fix permissions on /etc/jupyter as root +USER root +RUN fix-permissions /etc/jupyter/ + +############################################################################ +################# Dependency: jupyter/minimal-notebook ##################### +############################################################################ + +# Install all OS dependencies for fully functional notebook server +RUN apt-get update && apt-get install -yq --no-install-recommends \ + build-essential \ + emacs \ + git \ + inkscape \ + jed \ + libsm6 \ + libxext-dev \ + libxrender1 \ + lmodern \ + netcat \ + pandoc \ + python-dev \ + texlive-fonts-extra \ + texlive-fonts-recommended \ + texlive-generic-recommended \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-xetex \ + tzdata \ + unzip \ + nano \ + && rm -rf /var/lib/apt/lists/* + + +############################################################################ +################# Dependency: jupyter/scipy-notebook ####################### +############################################################################ + +# ffmpeg for matplotlib anim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +USER $NB_UID + +# Install Python 3 packages +RUN conda install --quiet --yes \ + 'conda-forge::blas=*=openblas' \ + 'ipywidgets=7.5*' \ + 'pandas=0.24*' \ + 'numexpr=2.6*' \ + 'matplotlib-base=3.1.*' \ + 'scipy=1.2*' \ + 'seaborn=0.9*' \ + 'scikit-learn=0.20*' \ + 'scikit-image=0.14*' \ + 'sympy=1.3*' \ + 'cython=0.29*' \ + 'patsy=0.5*' \ + 'statsmodels=0.9*' \ + 'cloudpickle=0.8*' \ + 'dill=0.2*' \ + 'dask=1.1.*' \ + 'numba=0.42*' \ + 'bokeh=1.0*' \ + 'sqlalchemy=1.3*' \ + 'hdf5=1.10*' \ + 'h5py=2.9*' \ + 'vincent=0.4.*' \ + 'beautifulsoup4=4.7.*' \ + 'protobuf=3.7.*' \ + 'xlrd' && \ + conda clean --all -f -y && \ + # Activate ipywidgets extension in the environment that runs the notebook server + jupyter nbextension enable --py widgetsnbextension --sys-prefix && \ + # Also activate ipywidgets extension for JupyterLab + # Check this URL for most recent compatibilities + # https://github.com/jupyter-widgets/ipywidgets/tree/master/packages/jupyterlab-manager + jupyter labextension install @jupyter-widgets/jupyterlab-manager@^1.0.1 --no-build && \ + jupyter labextension install jupyterlab_bokeh@1.0.0 --no-build && \ + jupyter lab build --dev-build=False && \ + npm cache clean --force && \ + rm -rf $CONDA_DIR/share/jupyter/lab/staging && \ + rm -rf /home/$NB_USER/.cache/yarn && \ + rm -rf /home/$NB_USER/.node-gyp && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Install facets which does not have a pip or conda package at the moment +RUN cd /tmp && \ + git clone https://github.com/PAIR-code/facets.git && \ + cd facets && \ + jupyter nbextension install facets-dist/ --sys-prefix && \ + cd && \ + rm -rf /tmp/facets && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Import matplotlib the first time to build the font cache. +ENV XDG_CACHE_HOME /home/$NB_USER/.cache/ +RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ + fix-permissions /home/$NB_USER + + +############################################################################ +################ Dependency: jupyter/datascience-notebook ################## +############################################################################ + +USER root + +# R pre-requisites +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + fonts-dejavu \ + gfortran \ + gcc && \ + rm -rf /var/lib/apt/lists/* + +# Julia dependencies +# install Julia packages in /opt/julia instead of $HOME +ENV JULIA_DEPOT_PATH=/opt/julia +ENV JULIA_PKGDIR=/opt/julia +ENV JULIA_VERSION=1.1.0 + +RUN mkdir /opt/julia-${JULIA_VERSION} && \ + cd /tmp && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \ + echo "80cfd013e526b5145ec3254920afd89bb459f1db7a2a3f21849125af20c05471 *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ + tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \ + rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz +RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia + +# Show Julia where conda libraries are \ +RUN mkdir /etc/julia && \ + echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ + # Create JULIA_PKGDIR \ + mkdir $JULIA_PKGDIR && \ + chown $NB_USER $JULIA_PKGDIR && \ + fix-permissions $JULIA_PKGDIR + +USER $NB_UID + +# R packages including IRKernel which gets installed globally. +RUN conda install --quiet --yes \ + 'rpy2=2.9*' \ + 'r-base=3.5.1' \ + 'r-irkernel=0.8*' \ + 'r-plyr=1.8*' \ + 'r-devtools=1.13*' \ + 'r-tidyverse=1.2*' \ + 'r-shiny=1.2*' \ + 'r-rmarkdown=1.11*' \ + 'r-forecast=8.2*' \ + 'r-rsqlite=2.1*' \ + 'r-reshape2=1.4*' \ + 'r-nycflights13=1.0*' \ + 'r-caret=6.0*' \ + 'r-rcurl=1.95*' \ + 'r-crayon=1.3*' \ + 'r-randomforest=4.6*' \ + 'r-htmltools=0.3*' \ + 'r-sparklyr=0.9*' \ + 'r-htmlwidgets=1.2*' \ + 'r-hexbin=1.27*' && \ + conda clean --all -f -y && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Add Julia packages. Only add HDF5 if this is not a test-only build since +# it takes roughly half the entire build time of all of the images on Travis +# to add this one package and often causes Travis to timeout. +# +# Install IJulia as jovyan and then move the kernelspec out +# to the system share location. Avoids problems with runtime UID change not +# taking effect properly on the .local folder in the jovyan home dir. +RUN julia -e 'import Pkg; Pkg.update()' && \ + (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ + julia -e "using Pkg; pkg\"add Gadfly RDatasets IJulia InstantiateFromURL\"; pkg\"precompile\"" && \ + # move kernelspec out of home \ + mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \ + chmod -R go+rx $CONDA_DIR/share/jupyter && \ + rm -rf $HOME/.local && \ + fix-permissions $JULIA_PKGDIR $CONDA_DIR/share/jupyter + + +############################################################################ +################ Dependency: jupyter/scipy-notebook ################## +############################################################################ + +USER root + +# ffmpeg for matplotlib anim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +USER $NB_UID + +# Install Python 3 packages +RUN conda install --quiet --yes \ + 'conda-forge::blas=*=openblas' \ + 'ipywidgets=7.5*' \ + 'pandas=0.24*' \ + 'numexpr=2.6*' \ + 'matplotlib-base=3.1.*' \ + 'scipy=1.2*' \ + 'seaborn=0.9*' \ + 'scikit-learn=0.20*' \ + 'scikit-image=0.14*' \ + 'sympy=1.3*' \ + 'cython=0.29*' \ + 'patsy=0.5*' \ + 'statsmodels=0.9*' \ + 'cloudpickle=0.8*' \ + 'dill=0.2*' \ + 'dask=1.1.*' \ + 'numba=0.42*' \ + 'bokeh=1.0*' \ + 'sqlalchemy=1.3*' \ + 'hdf5=1.10*' \ + 'h5py=2.9*' \ + 'vincent=0.4.*' \ + 'beautifulsoup4=4.7.*' \ + 'protobuf=3.7.*' \ + 'xlrd' && \ + conda clean --all -f -y && \ + # Activate ipywidgets extension in the environment that runs the notebook server + jupyter nbextension enable --py widgetsnbextension --sys-prefix && \ + # Also activate ipywidgets extension for JupyterLab + # Check this URL for most recent compatibilities + # https://github.com/jupyter-widgets/ipywidgets/tree/master/packages/jupyterlab-manager + jupyter labextension install @jupyter-widgets/jupyterlab-manager@^1.0.1 --no-build && \ + jupyter labextension install jupyterlab_bokeh@1.0.0 --no-build && \ + jupyter lab build --dev-build=False && \ + npm cache clean --force && \ + rm -rf $CONDA_DIR/share/jupyter/lab/staging && \ + rm -rf /home/$NB_USER/.cache/yarn && \ + rm -rf /home/$NB_USER/.node-gyp && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Install facets which does not have a pip or conda package at the moment +RUN cd /tmp && \ + git clone https://github.com/PAIR-code/facets.git && \ + cd facets && \ + jupyter nbextension install facets-dist/ --sys-prefix && \ + cd && \ + rm -rf /tmp/facets && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Import matplotlib the first time to build the font cache. +ENV XDG_CACHE_HOME /home/$NB_USER/.cache/ +RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ + fix-permissions /home/$NB_USER + + +############################################################################ +############### Dependency: jupyter/tensorflow-notebook #################### +############################################################################ + +# Install Tensorflow +RUN conda install --quiet --yes \ + 'tensorflow=1.13*' \ + 'keras=2.2*' && \ + conda clean --all -f -y && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# RUN pip install --upgrade tensorflow-gpu + +############################################################################ +########################## Dependency: pytorch ############################# +############################################################################ + +# Install PyTorch dependencies +RUN conda install --quiet --yes \ + pyyaml mkl mkl-include setuptools cmake cffi typing && \ + conda clean --all -f -y && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +# Install PyTorch +RUN conda install -y -c pytorch \ + cudatoolkit=10.1 \ + 'pytorch=1.3.1' \ + torchvision && \ + conda clean --all -f -y && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +############################################################################ +############################ Useful packages ############################### +############################################################################ + +COPY jupyter_notebook_config.json /etc/jupyter/ +RUN fix-permissions /home/$NB_USER + +# Switch back to jovyan to avoid accidental container runs as root +USER $NB_UID diff --git a/src/fix-permissions b/src/fix-permissions new file mode 100755 index 0000000..659b276 --- /dev/null +++ b/src/fix-permissions @@ -0,0 +1,35 @@ +#!/bin/bash +# set permissions on a directory +# after any installation, if a directory needs to be (human) user-writable, +# run this script on it. +# It will make everything in the directory owned by the group $NB_GID +# and writable by that group. +# Deployments that want to set a specific user id can preserve permissions +# by adding the `--group-add users` line to `docker run`. + +# uses find to avoid touching files that already have the right permissions, +# which would cause massive image explosion + +# right permissions are: +# group=$NB_GID +# AND permissions include group rwX (directory-execute) +# AND directories have setuid,setgid bits set + +set -e + +for d in "$@"; do + find "$d" \ + ! \( \ + -group $NB_GID \ + -a -perm -g+rwX \ + \) \ + -exec chgrp $NB_GID {} \; \ + -exec chmod g+rwX {} \; + # setuid,setgid *on directories only* + find "$d" \ + \( \ + -type d \ + -a ! -perm -6000 \ + \) \ + -exec chmod +6000 {} \; +done diff --git a/src/jupyter_notebook_config.json b/src/jupyter_notebook_config.json new file mode 100644 index 0000000..76d08e0 --- /dev/null +++ b/src/jupyter_notebook_config.json @@ -0,0 +1,6 @@ + +{ + "NotebookApp": { + "password": "sha1:e49e73b0eb0e:353c38d87d79527e57da1bba5f65046d0b376d95" + } +} diff --git a/src/jupyter_notebook_config.py b/src/jupyter_notebook_config.py new file mode 100644 index 0000000..0bb03b5 --- /dev/null +++ b/src/jupyter_notebook_config.py @@ -0,0 +1,55 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +from jupyter_core.paths import jupyter_data_dir +import subprocess +import os +import errno +import stat + +c = get_config() +c.NotebookApp.ip = '0.0.0.0' +c.NotebookApp.port = 8888 +c.NotebookApp.open_browser = False + +# https://github.com/jupyter/notebook/issues/3130 +c.FileContentsManager.delete_to_trash = False + +# Generate a self-signed certificate +if 'GEN_CERT' in os.environ: + dir_name = jupyter_data_dir() + pem_file = os.path.join(dir_name, 'notebook.pem') + try: + os.makedirs(dir_name) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(dir_name): + pass + else: + raise + + # Generate an openssl.cnf file to set the distinguished name + cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf') + if not os.path.isfile(cnf_file): + with open(cnf_file, 'w') as fh: + fh.write('''\ +[req] +distinguished_name = req_distinguished_name +[req_distinguished_name] +''') + + # Generate a certificate if one doesn't exist on disk + subprocess.check_call(['openssl', 'req', '-new', + '-newkey', 'rsa:2048', + '-days', '365', + '-nodes', '-x509', + '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated', + '-keyout', pem_file, + '-out', pem_file]) + # Restrict access to the file + os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) + c.NotebookApp.certfile = pem_file + +# Change default umask for all subprocesses of the notebook server if set in +# the environment +if 'NB_UMASK' in os.environ: + os.umask(int(os.environ['NB_UMASK'], 8)) diff --git a/src/start-notebook.sh b/src/start-notebook.sh new file mode 100755 index 0000000..fd55953 --- /dev/null +++ b/src/start-notebook.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then + # launched by JupyterHub, use single-user entrypoint + exec /usr/local/bin/start-singleuser.sh "$@" +elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then + . /usr/local/bin/start.sh jupyter lab "$@" +else + . /usr/local/bin/start.sh jupyter notebook "$@" +fi + diff --git a/src/start-singleuser.sh b/src/start-singleuser.sh new file mode 100755 index 0000000..f5ec3b0 --- /dev/null +++ b/src/start-singleuser.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# set default ip to 0.0.0.0 +if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then + NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" +fi + +# handle some deprecated environment variables +# from DockerSpawner < 0.8. +# These won't be passed from DockerSpawner 0.9, +# so avoid specifying --arg=empty-string +if [ ! -z "$NOTEBOOK_DIR" ]; then + NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_PORT" ]; then + NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_USER" ]; then + NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_COOKIE_NAME" ]; then + NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_BASE_URL" ]; then + NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_HUB_PREFIX" ]; then + NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_HUB_API_URL" ]; then + NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" +fi +if [ ! -z "$JUPYTER_ENABLE_LAB" ]; then + NOTEBOOK_BIN="jupyter labhub" +else + NOTEBOOK_BIN="jupyterhub-singleuser" +fi + +. /usr/local/bin/start.sh $NOTEBOOK_BIN $NOTEBOOK_ARGS "$@" + diff --git a/src/start.sh b/src/start.sh new file mode 100755 index 0000000..ad39498 --- /dev/null +++ b/src/start.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +set -e + +# Exec the specified command or fall back on bash +if [ $# -eq 0 ]; then + cmd=( "bash" ) +else + cmd=( "$@" ) +fi + +run-hooks () { + # Source scripts or run executable files in a directory + if [[ ! -d "$1" ]] ; then + return + fi + echo "$0: running hooks in $1" + for f in "$1/"*; do + case "$f" in + *.sh) + echo "$0: running $f" + source "$f" + ;; + *) + if [[ -x "$f" ]] ; then + echo "$0: running $f" + "$f" + else + echo "$0: ignoring $f" + fi + ;; + esac + done + echo "$0: done running hooks in $1" +} + +run-hooks /usr/local/bin/start-notebook.d + +# Handle special flags if we're root +if [ $(id -u) == 0 ] ; then + + # Only attempt to change the jovyan username if it exists + if id jovyan &> /dev/null ; then + echo "Set username to: $NB_USER" + usermod -d /home/$NB_USER -l $NB_USER jovyan + fi + + # Handle case where provisioned storage does not have the correct permissions by default + # Ex: default NFS/EFS (no auto-uid/gid) + if [[ "$CHOWN_HOME" == "1" || "$CHOWN_HOME" == 'yes' ]]; then + echo "Changing ownership of /home/$NB_USER to $NB_UID:$NB_GID with options '${CHOWN_HOME_OPTS}'" + chown $CHOWN_HOME_OPTS $NB_UID:$NB_GID /home/$NB_USER + fi + if [ ! -z "$CHOWN_EXTRA" ]; then + for extra_dir in $(echo $CHOWN_EXTRA | tr ',' ' '); do + echo "Changing ownership of ${extra_dir} to $NB_UID:$NB_GID with options '${CHOWN_EXTRA_OPTS}'" + chown $CHOWN_EXTRA_OPTS $NB_UID:$NB_GID $extra_dir + done + fi + + # handle home and working directory if the username changed + if [[ "$NB_USER" != "jovyan" ]]; then + # changing username, make sure homedir exists + # (it could be mounted, and we shouldn't create it if it already exists) + if [[ ! -e "/home/$NB_USER" ]]; then + echo "Relocating home dir to /home/$NB_USER" + mv /home/jovyan "/home/$NB_USER" + fi + # if workdir is in /home/jovyan, cd to /home/$NB_USER + if [[ "$PWD/" == "/home/jovyan/"* ]]; then + newcwd="/home/$NB_USER/${PWD:13}" + echo "Setting CWD to $newcwd" + cd "$newcwd" + fi + fi + + # Change UID of NB_USER to NB_UID if it does not match + if [ "$NB_UID" != $(id -u $NB_USER) ] ; then + echo "Set $NB_USER UID to: $NB_UID" + usermod -u $NB_UID $NB_USER + fi + + # Set NB_USER primary gid to NB_GID (after making the group). Set + # supplementary gids to NB_GID and 100. + if [ "$NB_GID" != $(id -g $NB_USER) ] ; then + echo "Add $NB_USER to group: $NB_GID" + groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}} + usermod -g $NB_GID -aG 100 $NB_USER + fi + + # Enable sudo if requested + if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then + echo "Granting $NB_USER sudo access and appending $CONDA_DIR/bin to sudo PATH" + echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook + fi + + # Add $CONDA_DIR/bin to sudo secure_path + sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path + + # Exec the command as NB_USER with the PATH and the rest of + # the environment preserved + run-hooks /usr/local/bin/before-notebook.d + echo "Executing the command: ${cmd[@]}" + exec sudo -E -H -u $NB_USER PATH=$PATH XDG_CACHE_HOME=/home/$NB_USER/.cache PYTHONPATH=${PYTHONPATH:-} "${cmd[@]}" +else + if [[ "$NB_UID" == "$(id -u jovyan)" && "$NB_GID" == "$(id -g jovyan)" ]]; then + # User is not attempting to override user/group via environment + # variables, but they could still have overridden the uid/gid that + # container runs as. Check that the user has an entry in the passwd + # file and if not add an entry. + STATUS=0 && whoami &> /dev/null || STATUS=$? && true + if [[ "$STATUS" != "0" ]]; then + if [[ -w /etc/passwd ]]; then + echo "Adding passwd file entry for $(id -u)" + cat /etc/passwd | sed -e "s/^jovyan:/nayvoj:/" > /tmp/passwd + echo "jovyan:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd + cat /tmp/passwd > /etc/passwd + rm /tmp/passwd + else + echo 'Container must be run with group "root" to update passwd file' + fi + fi + + # Warn if the user isn't going to be able to write files to $HOME. + if [[ ! -w /home/jovyan ]]; then + echo 'Container must be run with group "users" to update files' + fi + else + # Warn if looks like user want to override uid/gid but hasn't + # run the container as root. + if [[ ! -z "$NB_UID" && "$NB_UID" != "$(id -u)" ]]; then + echo 'Container must be run as root to set $NB_UID' + fi + if [[ ! -z "$NB_GID" && "$NB_GID" != "$(id -g)" ]]; then + echo 'Container must be run as root to set $NB_GID' + fi + fi + + # Warn if looks like user want to run in sudo mode but hasn't run + # the container as root. + if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then + echo 'Container must be run as root to grant sudo permissions' + fi + + # Execute the command + run-hooks /usr/local/bin/before-notebook.d + echo "Executing the command: ${cmd[@]}" + exec "${cmd[@]}" +fi + diff --git a/start-local.sh b/start-local.sh new file mode 100755 index 0000000..4ec3db7 --- /dev/null +++ b/start-local.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +docker-compose up --build -d diff --git a/stop-local.sh b/stop-local.sh new file mode 100755 index 0000000..d0b7406 --- /dev/null +++ b/stop-local.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +docker-compose down