From c833f1f9b70066ec85ac73658aeb39f84d1a6b7d Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Thu, 28 Jan 2021 11:19:37 +0100 Subject: [PATCH] build & push all image configurations (python-only mode) --- .build/Dockerfile | 115 ++++++++++++++++++++++++++++++++++++++--- .gitignore | 4 +- README.md | 15 +++--- build_push_all.sh | 54 +++++++++++++++++++ generate-Dockerfile.sh | 3 +- 5 files changed, 175 insertions(+), 16 deletions(-) create mode 100755 build_push_all.sh diff --git a/.build/Dockerfile b/.build/Dockerfile index b01d6df..1e26f9f 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -303,6 +303,107 @@ RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ USER $NB_UID +WORKDIR $HOME + + ############################################################################ + ################ Dependency: jupyter/datascience-notebook ################## + ############################################################################ + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +LABEL maintainer="Jupyter Project " + +# Set when building on Travis so that certain long-running build steps can +# be skipped to shorten build time. +ARG TEST_ONLY_BUILD + +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Julia installation +# Default values can be overridden at build time +# (ARGS are in lower case to distinguish them from ENV) +# Check https://julialang.org/downloads/ +ARG julia_version="1.5.3" +# SHA256 checksum +ARG julia_checksum="f190c938dd6fed97021953240523c9db448ec0a6760b574afd4e9924ab5615f1" + +# R pre-requisites +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + fonts-dejavu \ + gfortran \ + gcc && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Julia dependencies +# install Julia packages in /opt/julia instead of $HOME +ENV JULIA_DEPOT_PATH=/opt/julia \ + JULIA_PKGDIR=/opt/julia \ + JULIA_VERSION="${julia_version}" + +WORKDIR /tmp + +# hadolint ignore=SC2046 +RUN mkdir "/opt/julia-${JULIA_VERSION}" && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \ + echo "${julia_checksum} *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ + tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \ + rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" +RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia + +# Show Julia where conda libraries are \ +RUN mkdir /etc/julia && \ + echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ + # Create JULIA_PKGDIR \ + mkdir "${JULIA_PKGDIR}" && \ + chown "${NB_USER}" "${JULIA_PKGDIR}" && \ + fix-permissions "${JULIA_PKGDIR}" + +USER $NB_UID + +# R packages including IRKernel which gets installed globally. +RUN conda install --quiet --yes \ + 'r-base=4.0.3' \ + 'r-caret=6.0*' \ + 'r-crayon=1.3*' \ + 'r-devtools=2.3*' \ + 'r-forecast=8.13*' \ + 'r-hexbin=1.28*' \ + 'r-htmltools=0.5*' \ + 'r-htmlwidgets=1.5*' \ + 'r-irkernel=1.1*' \ + 'r-nycflights13=1.0*' \ + 'r-randomforest=4.6*' \ + 'r-rcurl=1.98*' \ + 'r-rmarkdown=2.6*' \ + 'r-rsqlite=2.2*' \ + 'r-shiny=1.5*' \ + 'r-tidyverse=1.3*' \ + 'rpy2=3.3*' && \ + conda clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Add Julia packages. Only add HDF5 if this is not a test-only build since +# it takes roughly half the entire build time of all of the images on Travis +# to add this one package and often causes Travis to timeout. +# +# Install IJulia as jovyan and then move the kernelspec out +# to the system share location. Avoids problems with runtime UID change not +# taking effect properly on the .local folder in the jovyan home dir. +RUN julia -e 'import Pkg; Pkg.update()' && \ + (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ + julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \ + # move kernelspec out of home \ + mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \ + chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \ + rm -rf "${HOME}/.local" && \ + fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" + WORKDIR $HOME ############################################################################ @@ -314,7 +415,7 @@ LABEL maintainer="Christoph Schranz " # Install Tensorflow, check compatibility here: https://www.tensorflow.org/install/gpu # installation via conda leads to errors in version 4.8.2 RUN pip install --upgrade pip && \ - pip install --no-cache-dir "tensorflow-gpu>=2.1.*" && \ + pip install --no-cache-dir "tensorflow==2.3.2" && \ pip install --no-cache-dir keras # Install PyTorch with dependencies @@ -361,13 +462,13 @@ RUN jupyter labextension install jupyterlab-drawio RUN jupyter labextension install jupyter-leaflet RUN jupyter labextension install jupyterlab-plotly@4.8.1 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager -RUN pip install --no-cache-dir jupyter-tabnine==1.0.2 && \ - jupyter nbextension install --py jupyter_tabnine && \ - jupyter nbextension enable --py jupyter_tabnine && \ - jupyter serverextension enable --py jupyter_tabnine +RUN pip install --no-cache-dir jupyter-tabnine==1.1.0 --user && \ + jupyter nbextension install --py jupyter_tabnine --user && \ + jupyter nbextension enable --py jupyter_tabnine --user && \ + jupyter serverextension enable --py jupyter_tabnine --user RUN pip install --no-cache-dir jupyter_contrib_nbextensions \ - jupyter_nbextensions_configurator rise && \ - jupyter nbextension enable codefolding/main + jupyter_nbextensions_configurator rise +# jupyter nbextension enable codefolding/main RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker RUN fix-permissions /home/$NB_USER diff --git a/.gitignore b/.gitignore index ff72562..8ab8c17 100644 --- a/.gitignore +++ b/.gitignore @@ -116,4 +116,6 @@ venv.bak/ src/jupyter_notebook_config.json .idea /Deployment-notes.md -/push_tag.sh +/push_tag_full.sh +/push_tag_python-only.sh +/push_tag_slim.sh diff --git a/README.md b/README.md index 6313c92..9c3e3d6 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The image of this repository is available on [Dockerhub](https://hub.docker.com/ docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.2_cuda-10.1_ubuntu-18.04_python-only ``` This starts an instance with of *GPU-Jupyter* the tag `v1.2_cuda-10.1_ubuntu-18.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8484`). - The default password is `asdf` which should be changed as described [below](#set-password). + The default password is `gpu-jupyter` (previously `asdf`) which should be changed as described [below](#set-password). Furthermore, data within the host's `data` directory is shared with the container. Other versions of GPU-Jupyter are available and listed on Dockerhub under [Tags](https://hub.docker.com/r/cschranz/gpu-jupyter/tags?page=1&ordering=last_updated). @@ -84,13 +84,13 @@ As soon as you have access to your GPU within Docker containers (make sure the command `docker run --gpus all nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 nvidia-smi` shows your GPU statistics), you can generate the Dockerfile, build and run it. The following commands will start *GPU-Jupyter* on [localhost:8848](http://localhost:8848) -with the default password `asdf`. +with the default password `gpu-jupyter` (previously `asdf`). ```bash git clone https://github.com/iot-salzburg/gpu-jupyter.git cd gpu-jupyter # generate a Dockerfile with python and without Julia and R - ./generate-Dockerfile.sh --no-datascience-notebook + ./generate-Dockerfile.sh --python-only docker build -t gpu-jupyter .build/ # will take a while docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes -e NB_UID="$(id -u)" -e NB_GID="$(id -g)" --user root --restart always --name gpu-jupyter_1 gpu-jupyter ``` @@ -147,7 +147,7 @@ Here the `docker-stack` `scipy-notebook` is used instead of `datascience-noteboo that comes with Julia and R. Moreover, none of the packages within `src/Dockerfile.usefulpackages` is installed. -* `--no-datascience-notebook`: As the name suggests, the `docker-stack` `datascience-notebook` +* `--python-only|--no-datascience-notebook`: As the name suggests, the `docker-stack` `datascience-notebook` is not installed on top of the `scipy-notebook`, but the packages within `src/Dockerfile.usefulpackages` are. @@ -170,7 +170,8 @@ If an essential package is missing in the default stack, please let us know! Please set a new password using `src/jupyter_notebook_config.json`. Therefore, hash your password in the form (password)(salt) using a sha1 hash generator, e.g., the sha1 generator of [sha1-online.com](http://www.sha1-online.com/). -The input with the default password `asdf` is appended by a arbitrary salt `e49e73b0eb0e` to `asdfe49e73b0eb0e` and should yield the hash string as shown in the config below. +The input with the default password `gpu-jupyter` (previously `asdf`) is concatenated by an arbitrary salt `3b4b6378355` to `gpu-jupyter3b4b6378355` and is hashed to `642693b20f0a33bcad27b94293d0ed7db3408322`. + **Never give away your own unhashed password!** Then update the config file as shown below and restart the service. @@ -178,7 +179,7 @@ Then update the config file as shown below and restart the service. ```json { "NotebookApp": { - "password": "sha1:e49e73b0eb0e:32edae7a5fd119045e699a0bd04f90819ca90cd6" + "password": "sha1:3b4b6378355:642693b20f0a33bcad27b94293d0ed7db3408322" } } ``` @@ -303,7 +304,7 @@ e.g., here it is **elk_datastack**. * **-r:** registry port is the port that is published by the registry service, default is `5000`. Now, *gpu-jupyter* will be accessible here on [localhost:8848](http://localhost:8848) -with the default password `asdf` and shares the network with the other data-source, i.e., +with the default password `gpu-jupyter` (previously `asdf`) and shares the network with the other data-source, i.e., all ports of the data-source will be accessible within *GPU-Jupyter*, even if they aren't routed it the source's `docker-compose` file. diff --git a/build_push_all.sh b/build_push_all.sh new file mode 100755 index 0000000..012b087 --- /dev/null +++ b/build_push_all.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +export TAGNAME="v1.3_cuda-10.1_ubuntu-18.04" + + +###################### build, run and push full image ########################## +echo +echo +echo "build, run and push full image with tag $TAGNAME." +bash generate-Dockerfile.sh +docker build -t cschranz/gpu-jupyter:$TAGNAME .build/ + +export IMG_ID=$(docker image ls | grep $TAGNAME | grep -v _python-only | grep -v _slim | head -1 | awk '{print $3}') +echo "push image with ID $IMG_ID and Tag $TAGNAME ." + +docker tag $IMG_ID cschranz/gpu-jupyter:$TAGNAME +docker rm -f gpu-jupyter_1 +docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root --restart always --name gpu-jupyter_1 cschranz/gpu-jupyter:$TAGNAME + +docker push cschranz/gpu-jupyter:$TAGNAME && +docker save cschranz/gpu-jupyter:$TAGNAME | gzip > ../gpu-jupyter_tag-$TAGNAME.tar.gz + + +###################### build and push slim image ########################## +echo +echo +echo "build and push slim image with tag ${TAGNAME}_slim." +bash generate-Dockerfile.sh --slim +docker build -t cschranz/gpu-jupyter:${TAGNAME}_slim .build/ + +export IMG_ID=$(docker image ls | grep ${TAGNAME}_slim | head -1 | awk '{print $3}') +echo "push image with ID $IMG_ID and Tag ${TAGNAME}_slim." + +docker tag $IMG_ID cschranz/gpu-jupyter:${TAGNAME}_slim +docker push cschranz/gpu-jupyter:${TAGNAME}_slim && +docker save cschranz/gpu-jupyter:${TAGNAME}_slim | gzip > ../gpu-jupyter_tag-${TAGNAME}_slim.tar.gz + + + +###################### build and push python-only image ########################## +echo +echo +echo "build and push slim image with tag ${TAGNAME}_slim." +bash generate-Dockerfile.sh --slim +docker build -t cschranz/gpu-jupyter:${TAGNAME}_slim .build/ + +export IMG_ID=$(docker image ls | grep ${TAGNAME}_slim | head -1 | awk '{print $3}') +echo "push image with ID $IMG_ID and Tag ${TAGNAME}_slim." + +docker tag $IMG_ID cschranz/gpu-jupyter:${TAGNAME}_slim +docker push cschranz/gpu-jupyter:${TAGNAME}_slim && +docker save cschranz/gpu-jupyter:${TAGNAME}_slim | gzip > ../gpu-jupyter_tag-${TAGNAME}_slim.tar.gz + diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 7dd5255..8d8f74b 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -10,6 +10,7 @@ export HEAD_COMMIT="703d8b2dcb886be2fe5aa4660a48fbcef647e7aa" while [[ "$#" -gt 0 ]]; do case $1 in -c|--commit) HEAD_COMMIT="$2"; shift;; --no-datascience-notebook) no_datascience_notebook=1;; + --python-only) no_datascience_notebook=1;; --no-useful-packages) no_useful_packages=1;; -s|--slim) no_datascience_notebook=1 && no_useful_packages=1;; *) echo "Unknown parameter passed: $1" && @@ -87,7 +88,7 @@ if [[ "$no_datascience_notebook" != 1 ]]; then " >> $DOCKERFILE cat $STACKS_DIR/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE else - echo "Set 'no-datascience-notebook', not installing the datascience-notebook with Julia and R." + echo "Set 'no-datascience-notebook' = 'python-only', not installing the datascience-notebook with Julia and R." fi # Note that the following step also installs the cudatoolkit, which is