updated underlying modules, improved description

2020-06-22 18:14:45 +02:00 · 2020-06-22 18:14:45 +02:00 · 099e818be2
commit 099e818be2
parent a4d5573698
5 changed files with 160 additions and 12 deletions
--- a/.build/Dockerfile
+++ b/.build/Dockerfile
@ -1,3 +1,5 @@
+# This adaptive Dockerfile is generated by 'generate-Dockerfile.sh' from parts within src/
+
 # Use NVIDIA CUDA as base image and run the same installation as in the other packages.
 # The version of cudatoolkit must match those of the base image, see Dockerfile.pytorch
 FROM nvidia/cuda:10.1-base-ubuntu18.04
@ -275,6 +277,102 @@ RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \

 USER $NB_UID

+WORKDIR $HOME
+
+  ############################################################################
+  ################ Dependency: jupyter/datascience-notebook ##################
+  ############################################################################
+  
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
+
+# Set when building on Travis so that certain long-running build steps can
+# be skipped to shorten build time.
+ARG TEST_ONLY_BUILD
+
+# Fix DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+# R pre-requisites
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    fonts-dejavu \
+    gfortran \
+    gcc && \
+    rm -rf /var/lib/apt/lists/*
+
+# Julia dependencies
+# install Julia packages in /opt/julia instead of $HOME
+ENV JULIA_DEPOT_PATH=/opt/julia
+ENV JULIA_PKGDIR=/opt/julia
+ENV JULIA_VERSION=1.4.1
+
+WORKDIR /tmp
+
+# hadolint ignore=SC2046
+RUN mkdir "/opt/julia-${JULIA_VERSION}" && \
+    wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \
+    echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
+    tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \
+    rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz"
+RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
+
+# Show Julia where conda libraries are \
+RUN mkdir /etc/julia && \
+    echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \
+    # Create JULIA_PKGDIR \
+    mkdir "${JULIA_PKGDIR}" && \
+    chown "${NB_USER}" "${JULIA_PKGDIR}" && \
+    fix-permissions "${JULIA_PKGDIR}"
+
+USER $NB_UID
+
+# R packages including IRKernel which gets installed globally.
+RUN conda install --quiet --yes \
+    'r-base=3.6.3' \
+    'r-caret=6.0*' \
+    'r-crayon=1.3*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
+    'r-hexbin=1.28*' \
+    'r-htmltools=0.4*' \
+    'r-htmlwidgets=1.5*' \
+    'r-irkernel=1.1*' \
+    'r-nycflights13=1.0*' \
+    'r-plyr=1.8*' \
+    'r-randomforest=4.6*' \
+    'r-rcurl=1.98*' \
+    'r-reshape2=1.4*' \
+    'r-rmarkdown=2.1*' \
+    'r-rsqlite=2.2*' \
+    'r-shiny=1.4*' \
+    'r-tidyverse=1.3*' \
+    'rpy2=3.1*' \
+    && \
+    conda clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+# Add Julia packages. Only add HDF5 if this is not a test-only build since
+# it takes roughly half the entire build time of all of the images on Travis
+# to add this one package and often causes Travis to timeout.
+#
+# Install IJulia as jovyan and then move the kernelspec out
+# to the system share location. Avoids problems with runtime UID change not
+# taking effect properly on the .local folder in the jovyan home dir.
+RUN julia -e 'import Pkg; Pkg.update()' && \
+    (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \
+    julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \
+    # move kernelspec out of home \
+    mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \
+    chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \
+    rm -rf "${HOME}/.local" && \
+    fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter"
+
 WORKDIR $HOME

 ############################################################################
@ -310,5 +408,54 @@ RUN conda clean --all -f -y && \
    fix-permissions $CONDA_DIR && \
    fix-permissions /home/$NB_USER

+  ############################################################################
+  ############################ Useful packages ###############################
+  ############################################################################
+  
+LABEL maintainer="Christoph Schranz <christoph.schranz@salzburgresearch.at>"
+
+# Update conda
+RUN conda update -n base conda -y
+
+USER root
+
+# Install elasticsearch libs
+USER root
+RUN apt-get update \
+ && curl -sL https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.1/elasticsearch-hadoop-6.8.1.jar
+RUN pip install --no-cache-dir elasticsearch==7.1.0
+
+RUN pip install --no-cache-dir ipyleaflet plotly==4.8.* "ipywidgets>=7.5"
+
+# Install important packages and Graphviz
+RUN set -ex \
+ && buildDeps=' \
+    graphviz==0.11 \
+' \
+ && apt-get update \
+ && apt-get -y install htop apt-utils graphviz libgraphviz-dev \
+ && pip install --no-cache-dir $buildDeps
+
+# Install various extensions
+RUN fix-permissions $CONDA_DIR
+RUN jupyter labextension install @jupyterlab/github
+RUN jupyter labextension install jupyterlab-drawio
+RUN jupyter labextension install jupyter-leaflet
+RUN jupyter labextension install jupyterlab-plotly@4.8.1
+RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
+RUN pip install --no-cache-dir jupyter-tabnine==1.0.2  && \
+  jupyter nbextension install --py jupyter_tabnine && \
+  jupyter nbextension enable --py jupyter_tabnine && \
+  jupyter serverextension enable --py jupyter_tabnine
+RUN pip install --no-cache-dir jupyter_contrib_nbextensions \
+ jupyter_nbextensions_configurator rise && \
+  jupyter nbextension enable codefolding/main
+RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker
+
+RUN fix-permissions /home/$NB_USER
+
+# Switch back to jovyan to avoid accidental container runs as root
+USER $NB_UID
+
 # Copy jupyter_notebook_config.json
 COPY jupyter_notebook_config.json /etc/jupyter/
--- a/.gitignore
+++ b/.gitignore
@ -109,6 +109,8 @@ venv.bak/
 !/data/.gitkeep
 *.save*
 *.envsubst
+*.tar
+*.tar.gz

 # Added config to hide hash of changed password
 src/jupyter_notebook_config.json
--- a/README.md
+++ b/README.md
@ -63,12 +63,12 @@ First of all, it is necessary to generate the `Dockerfile` based on the
 As soon as you have access to your GPU within Docker containers 
 (make sure the command `docker run --runtime nvidia nvidia/cuda:10.1-base-ubuntu18.04 nvidia-smi` shows your
 GPU statistics), you can generate a Dockerfile and build it via docker-compose.
-The two commands will start *GPU-Jupyter* on [localhost:1234](http://localhost:1234) with the default 
+The two commands will start *GPU-Jupyter* on [localhost:8848](http://localhost:8848) with the default 
 password `asdf`.

  ```bash
  ./generate-Dockerfile.sh
-  ./start-local.sh -p 1234  # where -p stands for the port, default 8888
+  ./start-local.sh -p 8848  # where -p stands for the port, default 8888
  ``` 

 ## Parameter
@ -228,7 +228,7 @@ Then re-generate and re-run the image, as closer described above:

 ```bash
 ./generate-Dockerfile.sh
-./start-local.sh -p [port]:8888  
+./start-local.sh -p 8848
 ```

 #### Update Docker-Stack
--- a/generate-Dockerfile.sh
+++ b/generate-Dockerfile.sh
@ -40,7 +40,9 @@ else
 fi

 # Write the contents into the DOCKERFILE and start with the header
-cat src/Dockerfile.header > $DOCKERFILE
+echo "# This adaptive Dockerfile is generated by 'generate-Dockerfile.sh' from parts within src/
+" > $DOCKERFILE
+cat src/Dockerfile.header >> $DOCKERFILE

 echo "
 ############################################################################
--- a/src/Dockerfile.usefulpackages
+++ b/src/Dockerfile.usefulpackages
@ -11,9 +11,7 @@ RUN apt-get update \
 && curl -sL https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.1/elasticsearch-hadoop-6.8.1.jar
 RUN pip install --no-cache-dir elasticsearch==7.1.0

-# Install rpy2 to share data between Python and R
-RUN conda install rpy2=2.9.4 plotly=4.4.1
-RUN conda install -c conda-forge ipyleaflet
+RUN pip install --no-cache-dir ipyleaflet plotly==4.8.* "ipywidgets>=7.5"

 # Install important packages and Graphviz
 RUN set -ex \
@ -25,19 +23,18 @@ RUN set -ex \
 && pip install --no-cache-dir $buildDeps

 # Install various extensions
+RUN fix-permissions $CONDA_DIR
 RUN jupyter labextension install @jupyterlab/github
 RUN jupyter labextension install jupyterlab-drawio
 RUN jupyter labextension install jupyter-leaflet
-RUN jupyter labextension install @jupyterlab/plotly-extension
+RUN jupyter labextension install jupyterlab-plotly@4.8.1
 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
 RUN pip install --no-cache-dir jupyter-tabnine==1.0.2  && \
  jupyter nbextension install --py jupyter_tabnine && \
  jupyter nbextension enable --py jupyter_tabnine && \
  jupyter serverextension enable --py jupyter_tabnine
-RUN fix-permissions $CONDA_DIR
-RUN conda install -c conda-forge jupyter_contrib_nbextensions && \
-  conda install -c conda-forge jupyter_nbextensions_configurator && \
-  conda install -c conda-forge rise && \
+RUN pip install --no-cache-dir jupyter_contrib_nbextensions \
+ jupyter_nbextensions_configurator rise && \
  jupyter nbextension enable codefolding/main
 RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker