From 099e818be2f9775459ebb38079341782f83bf6a3 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Mon, 22 Jun 2020 18:14:45 +0200 Subject: [PATCH] updated underlying modules, improved description --- .build/Dockerfile | 147 ++++++++++++++++++++++++++++++++++ .gitignore | 2 + README.md | 6 +- generate-Dockerfile.sh | 4 +- src/Dockerfile.usefulpackages | 13 ++- 5 files changed, 160 insertions(+), 12 deletions(-) diff --git a/.build/Dockerfile b/.build/Dockerfile index b49bd40..a2a2d22 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -1,3 +1,5 @@ +# This adaptive Dockerfile is generated by 'generate-Dockerfile.sh' from parts within src/ + # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cudatoolkit must match those of the base image, see Dockerfile.pytorch FROM nvidia/cuda:10.1-base-ubuntu18.04 @@ -275,6 +277,102 @@ RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ USER $NB_UID +WORKDIR $HOME + + ############################################################################ + ################ Dependency: jupyter/datascience-notebook ################## + ############################################################################ + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +LABEL maintainer="Jupyter Project " + +# Set when building on Travis so that certain long-running build steps can +# be skipped to shorten build time. +ARG TEST_ONLY_BUILD + +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# R pre-requisites +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + fonts-dejavu \ + gfortran \ + gcc && \ + rm -rf /var/lib/apt/lists/* + +# Julia dependencies +# install Julia packages in /opt/julia instead of $HOME +ENV JULIA_DEPOT_PATH=/opt/julia +ENV JULIA_PKGDIR=/opt/julia +ENV JULIA_VERSION=1.4.1 + +WORKDIR /tmp + +# hadolint ignore=SC2046 +RUN mkdir "/opt/julia-${JULIA_VERSION}" && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \ + echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ + tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \ + rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" +RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia + +# Show Julia where conda libraries are \ +RUN mkdir /etc/julia && \ + echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ + # Create JULIA_PKGDIR \ + mkdir "${JULIA_PKGDIR}" && \ + chown "${NB_USER}" "${JULIA_PKGDIR}" && \ + fix-permissions "${JULIA_PKGDIR}" + +USER $NB_UID + +# R packages including IRKernel which gets installed globally. +RUN conda install --quiet --yes \ + 'r-base=3.6.3' \ + 'r-caret=6.0*' \ + 'r-crayon=1.3*' \ + 'r-devtools=2.3*' \ + 'r-forecast=8.12*' \ + 'r-hexbin=1.28*' \ + 'r-htmltools=0.4*' \ + 'r-htmlwidgets=1.5*' \ + 'r-irkernel=1.1*' \ + 'r-nycflights13=1.0*' \ + 'r-plyr=1.8*' \ + 'r-randomforest=4.6*' \ + 'r-rcurl=1.98*' \ + 'r-reshape2=1.4*' \ + 'r-rmarkdown=2.1*' \ + 'r-rsqlite=2.2*' \ + 'r-shiny=1.4*' \ + 'r-tidyverse=1.3*' \ + 'rpy2=3.1*' \ + && \ + conda clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Add Julia packages. Only add HDF5 if this is not a test-only build since +# it takes roughly half the entire build time of all of the images on Travis +# to add this one package and often causes Travis to timeout. +# +# Install IJulia as jovyan and then move the kernelspec out +# to the system share location. Avoids problems with runtime UID change not +# taking effect properly on the .local folder in the jovyan home dir. +RUN julia -e 'import Pkg; Pkg.update()' && \ + (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ + julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \ + # move kernelspec out of home \ + mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \ + chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \ + rm -rf "${HOME}/.local" && \ + fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" + WORKDIR $HOME ############################################################################ @@ -310,5 +408,54 @@ RUN conda clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER + ############################################################################ + ############################ Useful packages ############################### + ############################################################################ + +LABEL maintainer="Christoph Schranz " + +# Update conda +RUN conda update -n base conda -y + +USER root + +# Install elasticsearch libs +USER root +RUN apt-get update \ + && curl -sL https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.1/elasticsearch-hadoop-6.8.1.jar +RUN pip install --no-cache-dir elasticsearch==7.1.0 + +RUN pip install --no-cache-dir ipyleaflet plotly==4.8.* "ipywidgets>=7.5" + +# Install important packages and Graphviz +RUN set -ex \ + && buildDeps=' \ + graphviz==0.11 \ +' \ + && apt-get update \ + && apt-get -y install htop apt-utils graphviz libgraphviz-dev \ + && pip install --no-cache-dir $buildDeps + +# Install various extensions +RUN fix-permissions $CONDA_DIR +RUN jupyter labextension install @jupyterlab/github +RUN jupyter labextension install jupyterlab-drawio +RUN jupyter labextension install jupyter-leaflet +RUN jupyter labextension install jupyterlab-plotly@4.8.1 +RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager +RUN pip install --no-cache-dir jupyter-tabnine==1.0.2 && \ + jupyter nbextension install --py jupyter_tabnine && \ + jupyter nbextension enable --py jupyter_tabnine && \ + jupyter serverextension enable --py jupyter_tabnine +RUN pip install --no-cache-dir jupyter_contrib_nbextensions \ + jupyter_nbextensions_configurator rise && \ + jupyter nbextension enable codefolding/main +RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker + +RUN fix-permissions /home/$NB_USER + +# Switch back to jovyan to avoid accidental container runs as root +USER $NB_UID + # Copy jupyter_notebook_config.json COPY jupyter_notebook_config.json /etc/jupyter/ diff --git a/.gitignore b/.gitignore index 7ade798..56548eb 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,8 @@ venv.bak/ !/data/.gitkeep *.save* *.envsubst +*.tar +*.tar.gz # Added config to hide hash of changed password src/jupyter_notebook_config.json diff --git a/README.md b/README.md index 1286404..e2d17e8 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,12 @@ First of all, it is necessary to generate the `Dockerfile` based on the As soon as you have access to your GPU within Docker containers (make sure the command `docker run --runtime nvidia nvidia/cuda:10.1-base-ubuntu18.04 nvidia-smi` shows your GPU statistics), you can generate a Dockerfile and build it via docker-compose. -The two commands will start *GPU-Jupyter* on [localhost:1234](http://localhost:1234) with the default +The two commands will start *GPU-Jupyter* on [localhost:8848](http://localhost:8848) with the default password `asdf`. ```bash ./generate-Dockerfile.sh - ./start-local.sh -p 1234 # where -p stands for the port, default 8888 + ./start-local.sh -p 8848 # where -p stands for the port, default 8888 ``` ## Parameter @@ -228,7 +228,7 @@ Then re-generate and re-run the image, as closer described above: ```bash ./generate-Dockerfile.sh -./start-local.sh -p [port]:8888 +./start-local.sh -p 8848 ``` #### Update Docker-Stack diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 535c4df..266161f 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -40,7 +40,9 @@ else fi # Write the contents into the DOCKERFILE and start with the header -cat src/Dockerfile.header > $DOCKERFILE +echo "# This adaptive Dockerfile is generated by 'generate-Dockerfile.sh' from parts within src/ +" > $DOCKERFILE +cat src/Dockerfile.header >> $DOCKERFILE echo " ############################################################################ diff --git a/src/Dockerfile.usefulpackages b/src/Dockerfile.usefulpackages index d79a9d6..280a23c 100644 --- a/src/Dockerfile.usefulpackages +++ b/src/Dockerfile.usefulpackages @@ -11,9 +11,7 @@ RUN apt-get update \ && curl -sL https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.1/elasticsearch-hadoop-6.8.1.jar RUN pip install --no-cache-dir elasticsearch==7.1.0 -# Install rpy2 to share data between Python and R -RUN conda install rpy2=2.9.4 plotly=4.4.1 -RUN conda install -c conda-forge ipyleaflet +RUN pip install --no-cache-dir ipyleaflet plotly==4.8.* "ipywidgets>=7.5" # Install important packages and Graphviz RUN set -ex \ @@ -25,19 +23,18 @@ RUN set -ex \ && pip install --no-cache-dir $buildDeps # Install various extensions +RUN fix-permissions $CONDA_DIR RUN jupyter labextension install @jupyterlab/github RUN jupyter labextension install jupyterlab-drawio RUN jupyter labextension install jupyter-leaflet -RUN jupyter labextension install @jupyterlab/plotly-extension +RUN jupyter labextension install jupyterlab-plotly@4.8.1 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager RUN pip install --no-cache-dir jupyter-tabnine==1.0.2 && \ jupyter nbextension install --py jupyter_tabnine && \ jupyter nbextension enable --py jupyter_tabnine && \ jupyter serverextension enable --py jupyter_tabnine -RUN fix-permissions $CONDA_DIR -RUN conda install -c conda-forge jupyter_contrib_nbextensions && \ - conda install -c conda-forge jupyter_nbextensions_configurator && \ - conda install -c conda-forge rise && \ +RUN pip install --no-cache-dir jupyter_contrib_nbextensions \ + jupyter_nbextensions_configurator rise && \ jupyter nbextension enable codefolding/main RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker