inherit PythonBuildPack from conda buildpack
sets up base env with conda
installs packages with pip
eliminates duplicate frozen envs for pip
py3/py2 split happens in one class, not separate classes
MODIFY
repo2docker/app.py
repo2docker/app.py
@@ -29,7 +29,7 @@ from traitlets.config import Application
from . import __version__
from .buildpacks import (
PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
- CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage,
+ CondaBuildPack, JuliaBuildPack, BaseImage,
RBuildPack
)
from .utils import (
@@ -67,7 +67,6 @@ class Repo2Docker(Application):
DockerBuildPack,
JuliaBuildPack,
CondaBuildPack,
- Python2BuildPack,
RBuildPack,
PythonBuildPack,
],

MODIFY
repo2docker/buildpacks/__init__.py
repo2docker/buildpacks/__init__.py
from .base import BuildPack, BaseImage
-from .python import Python2BuildPack, PythonBuildPack
+from .python import PythonBuildPack
from .conda import CondaBuildPack
from .julia import JuliaBuildPack
from .docker import DockerBuildPack

MODIFY
repo2docker/buildpacks/python/__init__.py
repo2docker/buildpacks/python/__init__.py
"""Generates Dockerfiles based on an input matrix based on Python."""
import os
-from ..base import BaseImage
+from ..conda import CondaBuildPack
-class PythonBuildPack(BaseImage):
- """Setup Python 3 for use with a repository."""
- def get_packages(self):
- """Return a list of the Python 3 core language packages to be installed
- via apt-get for this BuildPack.
+class PythonBuildPack(CondaBuildPack):
+ """Setup Python for use with a repository."""
- Note: The packages specified here are for the core Python3 language.
- Third party libraries are specified in other configuration files.
+ @property
+ def python_version(self):
+ if hasattr(self, '_python_version'):
+ return self._python_version
- """
- return super().get_packages().union({
- 'python3',
- 'python3-venv',
- 'python3-dev',
- })
-
- def get_env(self):
- """
- Return environment variables to be set.
-
- We set `VENV_PATH` to the virtual environment location and
- the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
-
- """
- return super().get_env() + [
- ("VENV_PATH", "${APP_BASE}/venv"),
- # Prefix to use for installing kernels and finding jupyter binary
- ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
- ]
-
- def get_path(self):
- """Return paths (including virtual environment path) to be added to
- the PATH environment variable.
-
- """
- return super().get_path() + [
- "${VENV_PATH}/bin"
- ]
-
- def get_build_script_files(self):
- """
- Dict of files to be copied to the container image for use in building.
-
- This is copied before the `build_scripts` & `assemble_scripts` are
- run, so can be executed from either of them.
-
- It's a dictionary where the key is the source file path in the host
- system, and the value is the destination file path inside the
- container image.
-
- This currently adds a frozen set of Python 3 requirements to the dict
- of files.
-
- """
- files = {
- 'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
- }
- files.update(super().get_build_script_files())
- return files
-
- def get_build_scripts(self):
- """
- Return series of build-steps common to all Python 3 repositories.
-
- All scripts here should be independent of contents of the repository.
-
- This sets up:
-
- - a directory for the virtual environment and its ownership by the
- notebook user
- - a Python 3 interpreter for the virtual environement
- - a Python 3 jupyter kernel including a base set of requirements
- - support for Jupyter widgets
- - support for JupyterLab
- - support for nteract
-
- """
- return super().get_build_scripts() + [
- (
- "root",
- r"""
- mkdir -p ${VENV_PATH} && \
- chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- python3 -m venv ${VENV_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \
- jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
- jupyter serverextension enable --py jupyterlab --sys-prefix && \
- jupyter serverextension enable nteract_on_jupyter --sys-prefix
- """
- )
- ]
+ try:
+ with open(self.binder_path('runtime.txt')) as f:
+ runtime = f.read().strip()
+ except FileNotFoundError:
+ runtime = 'python-' + self.major_pythons['3']
+ py_version_info = runtime.split('-', 1)[1].split('.')
+ py_version = ''
+ if len(py_version_info) == 1:
+ py_version = self.major_pythons[py_version_info[0]]
+ else:
+ # get major.minor
+ py_version = '.'.join(py_version_info[:2])
+ self._python_version = py_version
+ return self._python_version
def get_assemble_scripts(self):
"""Return series of build-steps specific to this repository.
@@ -112,28 +33,34 @@ class PythonBuildPack(BaseImage):
# If we have a runtime.txt & that's set to python-2.7,
# we will *not* install requirements.txt but will find &
# install a requirements3.txt file if it exists.
- # This way, when using python2 venv, requirements.txt will
- # be installed in the python2 venv, and requirements3.txt
- # will be installed in python3 venv. This is less of a
+ # This way, when using python2 env, requirements.txt will
+ # be installed in the python2 env, and requirements3.txt
+ # will be installed in python3 env. This is less of a
# surprise than requiring python2 to be requirements2.txt tho.
assemble_scripts = super().get_assemble_scripts()
setup_py = 'setup.py'
- try:
- with open(self.binder_path('runtime.txt')) as f:
- runtime = f.read().strip()
- except FileNotFoundError:
- runtime = 'python-3.5'
- if runtime == 'python-2.7':
- pip = "pip2"
- requirements_file = self.binder_path('requirements3.txt')
- else:
- pip = "pip3"
+ pip = 'pip'
+ if self.py2:
+ # using python 2 kernel,
+ # requirements3.txt allows installation in the notebook env
+ nb_requirements_file = self.binder_path('requirements3.txt')
+ if os.path.exists(nb_requirements_file):
+ assemble_scripts.append((
+ '${NB_USER}',
+ 'pip install --no-cache-dir -r "{}"'.format(nb_requirements_file)
+ ))
+ # subsequent `pip` calls should run in the kernel env
+ pip = '${KERNEL_PYTHON_PREFIX}/bin/pip'
+
+ # install requirements.txt in the kernel env
requirements_file = self.binder_path('requirements.txt')
if os.path.exists(requirements_file):
assemble_scripts.append((
'${NB_USER}',
- 'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
+ '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file)
))
+
+ # setup.py exists *and* binder dir is not used
if not os.path.exists('binder') and os.path.exists(setup_py):
assemble_scripts.append((
'${NB_USER}',
@@ -151,136 +78,10 @@ class PythonBuildPack(BaseImage):
if os.path.exists(runtime_txt):
with open(runtime_txt) as f:
runtime = f.read().strip()
- if runtime.startswith("python-3"):
+ if runtime.startswith("python-"):
return True
else:
return False
if not os.path.exists('binder') and os.path.exists(setup_py):
return True
return os.path.exists(requirements_txt)
-
-
-class Python2BuildPack(PythonBuildPack):
- """Setup Python 2 for use with a repository."""
- def get_packages(self):
- """Return a list of the Python 2 core language packages to be installed
- via apt-get for this BuildPack.
-
- Note: The packages specified here are for the core Python2 language.
- Third party libraries are specified in other configuration files.
-
- """
- return super().get_packages().union({
- 'python',
- 'python-dev',
- 'virtualenv'
- })
-
- def get_env(self):
- """
- Return environment variables to be set.
-
- We set `VENV_PATH` to the virtual environment location containing
- Python 2.
-
- """
- return super().get_env() + [
- ('VENV2_PATH', '${APP_BASE}/venv2')
- ]
-
- def get_path(self):
- """Return paths (including virtual environment path) to be added to
- the PATH environment variable.
-
- """
- return super().get_path() + [
- "${VENV2_PATH}/bin"
- ]
-
- def get_build_script_files(self):
- """
- Dict of files to be copied to the container image for use in building.
-
- This is copied before the `build_scripts` & `assemble_scripts` are
- run, so can be executed from either of them.
-
- It's a dictionary where the key is the source file path in the host
- system, and the value is the destination file path inside the
- container image.
-
- This currently adds a frozen set of Python 2 requirements to the dict
- of files.
-
- """
- files = {
- 'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
- }
- files.update(super().get_build_script_files())
- return files
-
- def get_build_scripts(self):
- """
- Return series of build-steps common to all Python 2 repositories.
-
- All scripts here should be independent of contents of the repository.
-
- This sets up:
-
- - a directory for the virtual environment and its ownership by the
- notebook user
- - a Python 2 interpreter for the virtual environement
- - a Python 2 jupyter kernel
-
- """
- return super().get_build_scripts() + [
- (
- "root",
- r"""
- mkdir -p ${VENV2_PATH} && \
- chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- virtualenv -p python2 ${VENV2_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \
- python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
- """
- )
- ]
-
- def get_assemble_scripts(self):
- """Return series of build-steps specific to this repository.
- """
- requirements_txt = self.binder_path('requirements.txt')
- assemble_scripts = super().get_assemble_scripts()
- if os.path.exists(requirements_txt):
- assemble_scripts.insert(0, (
- '${NB_USER}',
- 'pip2 install --no-cache-dir -r "{}"'.format(requirements_txt)
- ))
- return assemble_scripts
-
- def detect(self):
- """Check if current repo should be built with the Python 2 Build pack.
- """
- runtime_txt = self.binder_path('runtime.txt')
-
- if os.path.exists(runtime_txt):
- with open(runtime_txt) as f:
- runtime = f.read().strip()
- if runtime == 'python-2.7':
- return True
- elif runtime.startswith('python-2'):
- raise ValueError(
- "Only python-2.7 or python-3.x is supported in "
- "runtime.txt, not '{}'".format(runtime_txt))
- else:
- return False
- return False

DELETE
repo2docker/buildpacks/python/freeze.bash
-#!/bin/bash
-set -euo pipefail
-# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to
-# versions that are resolved at time of freezing.
-# Does the same for requirements2.txt to requirements2.frozen.txt...
-
-# cd to the directory where the freeze script is located
-if [[ ! -z "$(which realpath 2>/dev/null)" ]]; then
- realpath=realpath
-else
- realpath="readlink -f"
-fi
-
-cd $(dirname "$($realpath "$0")")
-
-
-function freeze-requirements {
- # Freeze a requirements file $2 into a frozen requirements file $3
- # Requires that a completely empty venv of appropriate version exist in $1
- PYTHON_VERSION="$1"
- REQUIREMENTS_FILE="$2"
- FROZEN_FILE="$3"
- if [[ $(echo ${PYTHON_VERSION} | cut -d. -f 1) == "2" ]]; then
- VENV=virtualenv
- else
- VENV=venv
- fi
-
- echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE}
- echo "# Frozen on $(date -u)" >> ${FROZEN_FILE}
- docker run --rm -v $PWD:/python -it python:${PYTHON_VERSION} \
- sh -c "
- python -m $VENV /venv
- /venv/bin/pip install -r /python/${REQUIREMENTS_FILE} &&
- /venv/bin/pip freeze | sort --ignore-case >> /python/${FROZEN_FILE}"
-}
-
-freeze-requirements 3.5 requirements.txt requirements.frozen.txt
-freeze-requirements 2.7 requirements2.txt requirements2.frozen.txt

DELETE
repo2docker/buildpacks/python/requirements.frozen.txt
-# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY
-# Frozen on Mon Apr 2 07:17:48 UTC 2018
-bleach==2.1.3
-decorator==4.2.1
-entrypoints==0.2.3
-html5lib==1.0.1
-ipykernel==4.8.2
-ipython-genutils==0.2.0
-ipython==6.2.1
-ipywidgets==7.1.1
-jedi==0.11.1
-Jinja2==2.10
-jsonschema==2.6.0
-jupyter-client==5.2.3
-jupyter-core==4.4.0
-jupyterlab-launcher==0.10.5
-jupyterlab==0.31.5
-MarkupSafe==1.0
-mistune==0.8.3
-nbconvert==5.3.1
-nbformat==4.4.0
-notebook==5.4.1
-nteract-on-jupyter==1.6.0
-pandocfilters==1.4.2
-parso==0.1.1
-pexpect==4.4.0
-pickleshare==0.7.4
-prompt-toolkit==1.0.15
-ptyprocess==0.5.2
-Pygments==2.2.0
-python-dateutil==2.7.2
-pyzmq==17.0.0
-Send2Trash==1.5.0
-simplegeneric==0.8.1
-six==1.11.0
-terminado==0.8.1
-testpath==0.3.1
-tornado==4.5.3
-traitlets==4.3.2
-wcwidth==0.1.7
-webencodings==0.5.1
-widgetsnbextension==3.1.4

DELETE
repo2docker/buildpacks/python/requirements.txt
-notebook==5.4.1
-tornado==4.5.3
-ipywidgets==7.1.1
-jupyterlab==0.31.5
-nteract_on_jupyter==1.6.0

DELETE
repo2docker/buildpacks/python/requirements2.frozen.txt
-# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY
-# Frozen on Mon Apr 2 07:19:00 UTC 2018
-backports-abc==0.5
-backports.shutil-get-terminal-size==1.0.0
-certifi==2018.1.18
-decorator==4.2.1
-enum34==1.1.6
-ipykernel==4.8.2
-ipython-genutils==0.2.0
-ipython==5.5.0
-jupyter-client==5.2.3
-jupyter-core==4.4.0
-pathlib2==2.3.0
-pexpect==4.4.0
-pickleshare==0.7.4
-prompt-toolkit==1.0.15
-ptyprocess==0.5.2
-Pygments==2.2.0
-python-dateutil==2.7.2
-pyzmq==17.0.0
-scandir==1.7
-simplegeneric==0.8.1
-singledispatch==3.4.0.3
-six==1.11.0
-tornado==4.5.3
-traitlets==4.3.2
-wcwidth==0.1.7

DELETE
repo2docker/buildpacks/python/requirements2.txt
-ipykernel==4.8.2
-tornado==4.5.3

inherit PythonBuildPack from conda buildpack

- sets up base env with conda
- installs packages with pip
- eliminates duplicate frozen envs for pip
- py3/py2 split happens in one class, not separate classes