Hello_User/gpu-jupyter
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feature: set sparse Dockerfile with Python interpreter only

Browse Source
This commit is contained in:
Christoph Schranz
2020-06-22 13:26:01 +02:00
parent da9c98689e
commit a4d5573698
10 changed files with 232 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
.build/Dockerfile
View File

@@ -2,7 +2,7 @@
# The version of cudatoolkit must match those of the base image, see Dockerfile.pytorch
FROM nvidia/cuda:10.1-base-ubuntu18.04
LABEL maintainer="Christoph Schranz <christoph.schranz@salzburgresearch.at>"
# The maintainers of subsequent sections may vary
# This is a concatenated Dockerfile, the maintainers of subsequent sections may vary.
############################################################################
#################### Dependency: jupyter/base-image ########################
@@ -11,15 +11,19 @@ LABEL maintainer="Christoph Schranz <christoph.schranz@salzburgresearch.at>"
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
# Ubuntu 18.04 (bionic)
# https://hub.docker.com/_/ubuntu/?tab=tags&name=bionic
ARG ROOT_CONTAINER=ubuntu:bionic-20200112@sha256:bc025862c3e8ec4a8754ea4756e33da6c41cba38330d7e324abd25c8e0b93300
# Ubuntu 20.04 (focal)
# https://hub.docker.com/_/ubuntu/?tab=tags&name=focal
# OS/ARCH: linux/amd64
ARG ROOT_CONTAINER=ubuntu:focal-20200423@sha256:238e696992ba9913d24cfc3727034985abd136e08ee3067982401acdc30cbf3f
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
ARG NB_USER="jovyan"
ARG NB_UID="1000"
ARG NB_GID="100"
# Fix DL4006
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
# Install all OS dependencies for notebook server that starts but lacks all
@@ -68,7 +72,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
chown $NB_USER:$NB_GID $CONDA_DIR && \
chmod g+w /etc/passwd && \
fix-permissions $HOME && \
fix-permissions "$(dirname $CONDA_DIR)"
fix-permissions $CONDA_DIR
USER $NB_UID
WORKDIR $HOME
@@ -79,19 +83,20 @@ RUN mkdir /home/$NB_USER/work && \
fix-permissions /home/$NB_USER
# Install conda as jovyan and check the md5 sum provided on the download site
ENV MINICONDA_VERSION=4.7.12.1 \
MINICONDA_MD5=81c773ff87af5cfac79ab862942ab6b3 \
CONDA_VERSION=4.7.12
ENV MINICONDA_VERSION=4.8.2 \
MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \
CONDA_VERSION=4.8.2
RUN cd /tmp && \
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
echo "${MINICONDA_MD5} *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
/bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \
WORKDIR /tmp
RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
/bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \
conda config --system --prepend channels conda-forge && \
conda config --system --set auto_update_conda false && \
conda config --system --set show_channel_urls true && \
conda config --system --set channel_priority strict && \
if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes python=$PYTHON_VERSION; fi && \
conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
conda install --quiet --yes conda && \
@@ -118,7 +123,7 @@ RUN conda install --quiet --yes 'tini=0.18.0' && \
RUN conda install --quiet --yes \
'notebook=6.0.3' \
'jupyterhub=1.1.0' \
'jupyterlab=1.2.5' && \
'jupyterlab=2.1.3' && \
conda clean --all -f -y && \
npm cache clean --force && \
jupyter notebook --generate-config && \
@@ -134,9 +139,7 @@ ENTRYPOINT ["tini", "-g", "--"]
CMD ["start-notebook.sh"]
# Copy local files as late as possible to avoid cache busting
COPY start.sh /usr/local/bin/
COPY start-notebook.sh /usr/local/bin/
COPY start-singleuser.sh /usr/local/bin/
COPY start.sh start-notebook.sh start-singleuser.sh /usr/local/bin/
COPY jupyter_notebook_config.py /etc/jupyter/
# Fix permissions on /etc/jupyter as root
@@ -146,6 +149,8 @@ RUN fix-permissions /etc/jupyter/
# Switch back to jovyan to avoid accidental container runs as root
USER $NB_UID
WORKDIR $HOME
############################################################################
################# Dependency: jupyter/minimal-notebook #####################
############################################################################
@@ -160,7 +165,8 @@ USER root
# Install all OS dependencies for fully functional notebook server
RUN apt-get update && apt-get install -yq --no-install-recommends \
build-essential \
emacs \
emacs-nox \
vim-tiny \
git \
inkscape \
jed \
@@ -173,7 +179,7 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
# ---- nbconvert dependencies ----
texlive-xetex \
texlive-fonts-recommended \
texlive-generic-recommended \
texlive-plain-generic \
# Optional dependency
texlive-fonts-extra \
# ----
@@ -196,40 +202,45 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
USER root
# ffmpeg for matplotlib anim
# ffmpeg for matplotlib anim & dvipng for latex labels
RUN apt-get update && \
apt-get install -y --no-install-recommends ffmpeg && \
apt-get install -y --no-install-recommends ffmpeg dvipng && \
rm -rf /var/lib/apt/lists/*
USER $NB_UID
# Install Python 3 packages
RUN conda install --quiet --yes \
'beautifulsoup4=4.8.*' \
'beautifulsoup4=4.9.*' \
'conda-forge::blas=*=openblas' \
'bokeh=1.4.*' \
'cloudpickle=1.2.*' \
'bokeh=2.0.*' \
'bottleneck=1.3.*' \
'cloudpickle=1.4.*' \
'cython=0.29.*' \
'dask=2.9.*' \
'dask=2.15.*' \
'dill=0.3.*' \
'h5py=2.10.*' \
'hdf5=1.10.*' \
'ipywidgets=7.5.*' \
'matplotlib-base=3.1.*' \
'ipympl=0.5.*'\
'matplotlib-base=3.2.*' \
# numba update to 0.49 fails resolving deps.
'numba=0.48.*' \
'numexpr=2.7.*' \
'pandas=0.25.*' \
'pandas=1.0.*' \
'patsy=0.5.*' \
'protobuf=3.11.*' \
'pytables=3.6.*' \
'scikit-image=0.16.*' \
'scikit-learn=0.22.*' \
'scipy=1.4.*' \
'seaborn=0.9.*' \
'seaborn=0.10.*' \
'sqlalchemy=1.3.*' \
'statsmodels=0.11.*' \
'sympy=1.5.*' \
'vincent=0.4.*' \
'xlrd' \
'widgetsnbextension=3.5.*'\
'xlrd=1.2.*' \
&& \
conda clean --all -f -y && \
# Activate ipywidgets extension in the environment that runs the notebook server
@@ -237,121 +248,34 @@ RUN conda install --quiet --yes \
# Also activate ipywidgets extension for JupyterLab
# Check this URL for most recent compatibilities
# https://github.com/jupyter-widgets/ipywidgets/tree/master/packages/jupyterlab-manager
jupyter labextension install @jupyter-widgets/jupyterlab-manager@^1.0.1 --no-build && \
jupyter labextension install jupyterlab_bokeh@1.0.0 --no-build && \
jupyter lab build && \
jupyter labextension install @jupyter-widgets/jupyterlab-manager@^2.0.0 --no-build && \
jupyter labextension install @bokeh/jupyter_bokeh@^2.0.0 --no-build && \
jupyter labextension install jupyter-matplotlib@^0.7.2 --no-build && \
jupyter lab build -y && \
jupyter lab clean -y && \
npm cache clean --force && \
rm -rf $CONDA_DIR/share/jupyter/lab/staging && \
rm -rf /home/$NB_USER/.cache/yarn && \
rm -rf /home/$NB_USER/.node-gyp && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
rm -rf "/home/${NB_USER}/.cache/yarn" && \
rm -rf "/home/${NB_USER}/.node-gyp" && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Install facets which does not have a pip or conda package at the moment
RUN cd /tmp && \
git clone https://github.com/PAIR-code/facets.git && \
cd facets && \
jupyter nbextension install facets-dist/ --sys-prefix && \
cd && \
WORKDIR /tmp
RUN git clone https://github.com/PAIR-code/facets.git && \
jupyter nbextension install facets/facets-dist/ --sys-prefix && \
rm -rf /tmp/facets && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Import matplotlib the first time to build the font cache.
ENV XDG_CACHE_HOME /home/$NB_USER/.cache/
ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/"
RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \
fix-permissions /home/$NB_USER
fix-permissions "/home/${NB_USER}"
USER $NB_UID
############################################################################
################ Dependency: jupyter/datascience-notebook ##################
############################################################################
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
# Set when building on Travis so that certain long-running build steps can
# be skipped to shorten build time.
ARG TEST_ONLY_BUILD
USER root
# R pre-requisites
RUN apt-get update && \
apt-get install -y --no-install-recommends \
fonts-dejavu \
gfortran \
gcc && \
rm -rf /var/lib/apt/lists/*
# Julia dependencies
# install Julia packages in /opt/julia instead of $HOME
ENV JULIA_DEPOT_PATH=/opt/julia
ENV JULIA_PKGDIR=/opt/julia
ENV JULIA_VERSION=1.3.1
RUN mkdir /opt/julia-${JULIA_VERSION} && \
cd /tmp && \
wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
echo "faa707c8343780a6fe5eaf13490355e8190acf8e2c189b9e7ecbddb0fa2643ad *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
# Show Julia where conda libraries are \
RUN mkdir /etc/julia && \
echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \
# Create JULIA_PKGDIR \
mkdir $JULIA_PKGDIR && \
chown $NB_USER $JULIA_PKGDIR && \
fix-permissions $JULIA_PKGDIR
USER $NB_UID
# R packages including IRKernel which gets installed globally.
RUN conda install --quiet --yes \
'r-base=3.6.2' \
'r-caret=6.0*' \
'r-crayon=1.3*' \
'r-devtools=2.2*' \
'r-forecast=8.10*' \
'r-hexbin=1.28*' \
'r-htmltools=0.4*' \
'r-htmlwidgets=1.5*' \
'r-irkernel=1.1*' \
'r-nycflights13=1.0*' \
'r-plyr=1.8*' \
'r-randomforest=4.6*' \
'r-rcurl=1.98*' \
'r-reshape2=1.4*' \
'r-rmarkdown=2.1*' \
'r-rsqlite=2.1*' \
'r-shiny=1.3*' \
'r-tidyverse=1.3*' \
'rpy2=3.1*' \
&& \
conda clean --all -f -y && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
# Add Julia packages. Only add HDF5 if this is not a test-only build since
# it takes roughly half the entire build time of all of the images on Travis
# to add this one package and often causes Travis to timeout.
#
# Install IJulia as jovyan and then move the kernelspec out
# to the system share location. Avoids problems with runtime UID change not
# taking effect properly on the .local folder in the jovyan home dir.
RUN julia -e 'import Pkg; Pkg.update()' && \
(test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \
julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \
# move kernelspec out of home \
mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \
chmod -R go+rx $CONDA_DIR/share/jupyter && \
rm -rf $HOME/.local && \
fix-permissions $JULIA_PKGDIR $CONDA_DIR/share/jupyter
WORKDIR $HOME
############################################################################
########################## Dependency: gpulibs #############################
@@ -359,12 +283,16 @@ RUN julia -e 'import Pkg; Pkg.update()' && \
LABEL maintainer="Christoph Schranz <christoph.schranz@salzburgresearch.at>"
# Install Tensorflow, check compatibility here: https://www.tensorflow.org/install/gpu
RUN conda install --quiet --yes \
'tensorflow-gpu=2.1*' \
'keras-gpu' && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
# Install Tensorflow, check compatibility here: https://www.tensorflow.org/install/gpu
# installation via conda leads to errors in version 4.8.2
#RUN conda install --quiet --yes \
# 'tensorflow-gpu=2.1*' \
# 'keras-gpu' && \
# fix-permissions $CONDA_DIR && \
# fix-permissions /home/$NB_USER
RUN pip install --upgrade pip && \
pip install --no-cache-dir "tensorflow-gpu>=2.1.*" && \
pip install --no-cache-dir keras
# Install PyTorch with dependencies
RUN conda install --quiet --yes \
@@ -382,57 +310,5 @@ RUN conda clean --all -f -y && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
############################################################################
############################ Useful packages ###############################
############################################################################
LABEL maintainer="Christoph Schranz <christoph.schranz@salzburgresearch.at>"
# Update conda
RUN conda update -n base conda -y
USER root
# Install elasticsearch libs
USER root
RUN apt-get update \
&& curl -sL https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.1/elasticsearch-hadoop-6.8.1.jar
RUN pip install --no-cache-dir elasticsearch==7.1.0
# Install rpy2 to share data between Python and R
RUN conda install rpy2=2.9.4 plotly=4.4.1
RUN conda install -c conda-forge ipyleaflet
# Install important packages and Graphviz
RUN set -ex \
&& buildDeps=' \
graphviz==0.11 \
' \
&& apt-get update \
&& apt-get -y install htop apt-utils graphviz libgraphviz-dev \
&& pip install --no-cache-dir $buildDeps
# Install various extensions
RUN jupyter labextension install @jupyterlab/github
RUN jupyter labextension install jupyterlab-drawio
RUN jupyter labextension install jupyter-leaflet
RUN jupyter labextension install @jupyterlab/plotly-extension
RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
RUN pip install --no-cache-dir jupyter-tabnine==1.0.2 && \
jupyter nbextension install --py jupyter_tabnine && \
jupyter nbextension enable --py jupyter_tabnine && \
jupyter serverextension enable --py jupyter_tabnine
RUN fix-permissions $CONDA_DIR
RUN conda install -c conda-forge jupyter_contrib_nbextensions && \
conda install -c conda-forge jupyter_nbextensions_configurator && \
conda install -c conda-forge rise && \
jupyter nbextension enable codefolding/main
RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker
RUN fix-permissions /home/$NB_USER
# Switch back to jovyan to avoid accidental container runs as root
USER $NB_UID
# Copy jupyter_notebook_config.json
COPY jupyter_notebook_config.json /etc/jupyter/