support separate pip requirements.txt in base env
require base env to be a lockfile, environment.yml no longer allowed
separate `requirements.py-x.y.txt` can be processed post-install, if present
use this for Python 3.5 env, which has some dependencies only from pip (newer versions than the last py35 build on conda-forge)
MODIFY
repo2docker/buildpacks/conda/__init__.py
repo2docker/buildpacks/conda/__init__.py
@@ -23,13 +23,17 @@ class CondaBuildPack(BaseImage):
"""
- # The kernel environment file, if any.
+ # The kernel conda environment file, if any.
# As an absolute path within the container.
_kernel_environment_file = ""
+ # extra pip requirements.txt for the kernel
+ _kernel_requirements_file = ""
- # The notebook server environment file, if any.
+ # The notebook server environment file.
# As an absolute path within the container.
_nb_environment_file = ""
+ # extra pip requirements.txt for the notebook env
+ _nb_requirements_file = ""
def get_build_env(self):
"""Return environment variables to be set.
@@ -38,18 +42,28 @@ class CondaBuildPack(BaseImage):
the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
"""
+ if not self._nb_environment_file:
+ # get_build_scripts locates requirements/environment files
+ self.get_build_scripts()
+
env = super().get_build_env() + [
("CONDA_DIR", "${APP_BASE}/conda"),
("NB_PYTHON_PREFIX", "${CONDA_DIR}/envs/notebook"),
("NB_ENVIRONMENT_FILE", self._nb_environment_file),
]
- if self.py2:
+ if self._nb_requirements_file:
+ env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file))
+
+ if self._kernel_environment_file:
+ # if kernel environment file is separate
env.extend(
[
("KERNEL_PYTHON_PREFIX", "${CONDA_DIR}/envs/kernel"),
("KERNEL_ENVIRONMENT_FILE", self._kernel_environment_file),
]
)
+ if self._kernel_requirements_file:
+ env.append(("KERNEL_REQUIREMENTS_FILE", self._kernel_requirements_file))
else:
env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}"))
return env
@@ -95,7 +109,7 @@ class CondaBuildPack(BaseImage):
r"""
TIMEFORMAT='time: %3R' \
bash -c 'time /tmp/install-miniforge.bash' && \
- rm /tmp/install-miniforge.bash ${NB_ENVIRONMENT_FILE}
+ rm -rf /tmp/install-miniforge.bash /tmp/env
""",
)
]
@@ -129,24 +143,37 @@ class CondaBuildPack(BaseImage):
# If no version is specified or no matching X.Y version is found,
# the default base environment is used.
frozen_name = "environment.lock"
+ pip_frozen_name = "requirements.txt"
if py_version:
if self.py2:
# python 2 goes in a different env
files[
"conda/environment.py-2.7.lock"
- ] = self._kernel_environment_file = "/tmp/kernel-environment.lock"
+ ] = self._kernel_environment_file = "/tmp/env/kernel-environment.lock"
+ # additional pip requirements for kernel env
+ if os.path.exists(os.path.join(HERE, "requirements.py-2.7.txt")):
+ files[
+ "conda/requirements.py-2.7.txt"
+ ] = (
+ self._kernel_requirements_file
+ ) = "/tmp/env/kernel-requirements.txt"
else:
- for ext in [".lock", ".frozen.yml"]:
- py_frozen_name = f"environment.py-{py_version}{ext}"
+ py_frozen_name = f"environment.py-{py_version}.lock"
if os.path.exists(os.path.join(HERE, py_frozen_name)):
frozen_name = py_frozen_name
- break
+ pip_frozen_name = f"requirements.py-{py_version}.txt"
if not frozen_name:
self.log.warning(f"No frozen env for {py_version}")
- _, frozen_ext = os.path.splitext(frozen_name)
files[
"conda/" + frozen_name
- ] = self._nb_environment_file = f"/tmp/environment{frozen_ext}"
+ ] = self._nb_environment_file = "/tmp/env/environment.lock"
+
+ # add requirements.txt, if present
+ if os.path.exists(os.path.join(HERE, pip_frozen_name)):
+ files[
+ "conda/" + pip_frozen_name
+ ] = self._nb_requirements_file = "/tmp/env/requirements.txt"
+
files.update(super().get_build_script_files())
return files

DELETE
repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml
-# py3.5 conda packages are no longer being updated as of 12/2018
-# this file is hand-maintained now
-# and should only receive security fixes, no new features
-name: r2d
-channels:
- - conda-forge
- - defaults
- - conda-forge/label/broken
-dependencies:
- - backcall=0.1.0=py_0
- - bleach=3.0.2=py_0
- - bzip2=1.0.6=h470a237_2
- - ca-certificates=2018.10.15=ha4d7672_0
- - certifi=2018.8.24=py35_1001
- - decorator=4.3.0=py_0
- - defusedxml=0.5.0=py_1
- - entrypoints=0.2.3=py35_2
- - gmp=6.1.2=hfc679d8_0
- - ipykernel=5.1.0=pyh24bf2e0_0
- - ipython=7.0.1=py35h24bf2e0_0
- - ipython_genutils=0.2.0=py_1
- - ipywidgets=7.2.1=py35_1
- - jedi=0.12.1=py35_0
- - jinja2=2.10=py_1
- - jsonschema=2.6.0=py35_2
- - jupyter_client=5.2.3=py_1
- - jupyter_core=4.4.0=py_0
- - jupyterlab=0.34.9=py35_0
- - jupyterlab_launcher=0.13.1=py_2
- - libffi=3.2.1=hfc679d8_5
- - libgcc-ng=7.2.0=hdf63c60_3
- - libsodium=1.0.16=h470a237_1
- - libstdcxx-ng=7.2.0=hdf63c60_3
- - markupsafe=1.0=py35h470a237_1
- - mistune=0.8.3=py35h470a237_2
- - nbconvert=5.4.0=1
- - nbformat=4.4.0=py_1
- - ncurses=6.1=hfc679d8_1
- - openssl=1.0.2p=h470a237_1
- - pandoc=1.19.2=0
- - pandocfilters=1.4.2=py_1
- - parso=0.3.1=py_0
- - pexpect=4.6.0=py35_0
- - pickleshare=0.7.5=py35_0
- - pip=18.0=py35_1001
- - prometheus_client=0.4.2=py_0
- - prompt_toolkit=2.0.7=py_0
- - ptyprocess=0.6.0=py35_1000
- - pygments=2.2.0=py_1
- - python=3.5.5=h5001a0f_2
- - python-dateutil=2.7.5=py_0
- - pyzmq=17.1.2=py35hae99301_0
- - readline=7.0=haf1bffa_1
- - send2trash=1.5.0=py_0
- - setuptools=40.4.3=py35_0
- - simplegeneric=0.8.1=py_1
- - six=1.11.0=py35_1
- - sqlite=3.25.3=hb1c47c0_0
- - terminado=0.8.1=py35_1
- - testpath=0.3.1=py35_1
- - tk=8.6.9=ha92aebf_0
- - tornado=5.1.1=py35h470a237_0
- - traitlets=4.3.2=py35_0
- - wcwidth=0.1.7=py_1
- - webencodings=0.5.1=py_1
- - wheel=0.32.0=py35_1000
- - widgetsnbextension=3.2.1=py35_1
- - xz=5.2.4=h470a237_1
- - zeromq=4.2.5=hfc679d8_6
- - zlib=1.2.11=h470a237_3
- - pip:
- - alembic==1.0.3
- - async-generator==1.10
- - chardet==3.0.4
- - idna==2.7
- - jupyterhub==0.9.4
- - mako==1.0.7
- - notebook==5.7.8
- - nteract-on-jupyter==1.9.6
- - pamela==0.3.0
- - python-editor==1.0.3
- - python-oauth2==1.1.0
- - requests==2.20.1
- - sqlalchemy==1.2.14
- - urllib3==1.24.1
-prefix: /opt/conda/envs/r2d

ADD
repo2docker/buildpacks/conda/environment.py-3.5.lock
+# py3.5 conda packages are no longer being updated as of 12/2018
+# this file is hand-maintained now
+# and should only receive security fixes, no new features
+@EXPLICIT
+https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-main.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2018.10.15-ha4d7672_0.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-7.2.0-hdf63c60_3.conda
+https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-7.2.0-hdf63c60_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h470a237_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/gmp-6.1.2-hfc679d8_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-hfc679d8_5.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.16-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hfc679d8_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/openssl-1.0.2p-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h470a237_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pandoc-1.19.2-0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-haf1bffa_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.9-ha92aebf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.2.5-hfc679d8_6.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.25.3-hb1c47c0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python-3.5.5-h5001a0f_2.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/backcall-0.1.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/certifi-2018.8.24-py35_1001.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/decorator-4.3.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.5.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/entrypoints-0.2.3-py35_2.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jsonschema-2.6.0-py35_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.0-py35h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/mistune-0.8.3-py35h470a237_2.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.4.2-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/parso-0.3.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pickleshare-0.7.5-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.4.2-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ptyprocess-0.6.0-py35_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pyzmq-17.1.2-py35hae99301_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/send2trash-1.5.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/simplegeneric-0.8.1-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/six-1.11.0-py35_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/testpath-0.3.1-py35_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tornado-5.1.1-py35h470a237_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.1.7-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jedi-0.12.1-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pexpect-4.6.0-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/setuptools-40.4.3-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/terminado-0.8.1-py35_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/traitlets-4.3.2-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/bleach-3.0.2-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/jupyter_core-4.4.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pygments-2.2.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/wheel-0.32.0-py35_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/jupyter_client-5.2.3-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/nbformat-4.4.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pip-18.0-py35_1001.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-2.0.7-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ipython-7.0.1-py35h24bf2e0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/label/broken/noarch/nbconvert-5.4.0-1.tar.bz2
+https://conda.anaconda.org/conda-forge/label/broken/noarch/ipykernel-5.1.0-pyh24bf2e0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/notebook-5.7.0-py35_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/jupyterlab_launcher-0.13.1-py_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/widgetsnbextension-3.2.1-py35_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ipywidgets-7.2.1-py35_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jupyterlab-0.34.9-py35_0.tar.bz2

MODIFY
repo2docker/buildpacks/conda/install-miniforge.bash
repo2docker/buildpacks/conda/install-miniforge.bash
@@ -47,31 +47,31 @@ time mamba install -y mamba==${MAMBA_VERSION}
echo "installing notebook env:"
cat "${NB_ENVIRONMENT_FILE}"
-if [[ "${NB_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
- create="mamba create"
-else
- create="mamba env create"
-fi
-
-time $create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
+time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
+if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then
+ echo "installing pip requirements"
+ cat "${NB_REQUIREMENTS_FILE}"
+ ${NB_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${NB_REQUIREMENTS_FILE}"
+fi
# empty conda history file,
# which seems to result in some effective pinning of packages in the initial env,
# which we don't intend.
# this file must not be *removed*, however
echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history
-if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" && -f "${KERNEL_ENVIRONMENT_FILE}" ]]; then
+if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
# install kernel env and register kernelspec
echo "installing kernel env:"
cat "${KERNEL_ENVIRONMENT_FILE}"
- if [[ "${KERNEL_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
- create="mamba create"
- else
- create="mamba env create"
+ time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
+
+ if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then
+ echo "installing pip requirements for kernel"
+ cat "${KERNEL_REQUIREMENTS_FILE}"
+ ${KERNEL_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${KERNEL_REQUIREMENTS_FILE}"
fi
- time $create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
mamba list -p ${KERNEL_PYTHON_PREFIX}

ADD
repo2docker/buildpacks/conda/requirements.py-3.5.txt
+# pip-installed packages for Python 3.5
+# these should only be packages
+# whose versions are too new to have builds available for Python 3.5
+alembic==1.0.3
+async-generator==1.10
+chardet==3.0.4
+idna==2.7
+jupyterhub==0.9.4
+mako==1.0.7
+notebook==5.7.8
+nteract-on-jupyter==1.9.6
+pamela==0.3.0
+python-editor==1.0.3
+python-oauth2==1.1.0
+requests==2.20.1
+sqlalchemy==1.2.14
+urllib3==1.24.1

support separate pip requirements.txt in base env

- require base env to be a lockfile, environment.yml no longer allowed
- separate `requirements.py-x.y.txt` can be processed post-install, if present
- use this for Python 3.5 env, which has some dependencies only from pip (newer versions than the last py35 build on conda-forge)