[conda] create separate frozen envs for Python versions
this will allow us to create different base envs based on Python major.minor versions
MODIFY
repo2docker/buildpacks/conda/environment.frozen.yml
repo2docker/buildpacks/conda/environment.frozen.yml
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
name: r2d
channels:
- conda-forge
@@ -11,13 +13,13 @@ dependencies:
- gmp=6.1.2=0
- html5lib=1.0.1=py_0
- ipykernel=4.7.0=py36_0
-- ipython=6.2.1=py36_0
+- ipython=6.2.1=py36_1
- ipython_genutils=0.2.0=py36_0
- ipywidgets=6.0.1=py36_0
- jedi=0.10.2=py36_0
- jinja2=2.10=py36_0
- jsonschema=2.6.0=py36_0
-- jupyter_client=5.1.0=py36_0
+- jupyter_client=5.2.0=py36_0
- jupyter_core=4.4.0=py_0
- jupyterlab=0.30.6=py36_0
- jupyterlab_launcher=0.6.0=py36_0
@@ -31,23 +33,23 @@ dependencies:
- openssl=1.0.2n=0
- pandoc=2.0.5=0
- pandocfilters=1.4.1=py36_0
-- pexpect=4.3.0=py36_0
+- pexpect=4.3.1=py36_0
- pickleshare=0.7.4=py36_0
-- pip=9.0.1=py36_0
+- pip=9.0.1=py36_1
- prompt_toolkit=1.0.15=py36_0
- ptyprocess=0.5.2=py36_0
- pygments=2.2.0=py36_0
-- python=3.6.1=3
+- python=3.6.4=0
- python-dateutil=2.6.1=py36_0
- pyzmq=16.0.2=py36_2
-- readline=6.2=0
+- readline=7.0=0
- setuptools=38.2.4=py36_0
- simplegeneric=0.8.1=py36_0
- six=1.11.0=py36_1
-- sqlite=3.13.0=1
+- sqlite=3.20.1=2
- terminado=0.8.1=py36_0
- testpath=0.3.1=py36_0
-- tk=8.5.19=2
+- tk=8.6.7=0
- tornado=4.5.2=py36_0
- traitlets=4.3.2=py36_0
- wcwidth=0.1.7=py36_0

ADD
repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml
+# AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:29:14 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- backports=1.0=py27_1
+- backports.shutil_get_terminal_size=1.0.0=py_3
+- backports_abc=0.5=py27_0
+- bleach=2.0.0=py27_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py27_0
+- configparser=3.5.0=py27_0
+- decorator=4.1.2=py27_0
+- entrypoints=0.2.3=py27_1
+- enum34=1.1.6=py27_1
+- functools32=3.2.3.2=py27_1
+- futures=3.2.0=py27_0
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py27_0
+- ipython=5.5.0=py27_0
+- ipython_genutils=0.2.0=py27_0
+- ipywidgets=6.0.1=py27_0
+- jinja2=2.10=py27_0
+- jsonschema=2.6.0=py27_0
+- jupyter_client=5.2.0=py27_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py27_0
+- jupyterlab_launcher=0.6.0=py27_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py27_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py27_0
+- ncurses=5.9=10
+- notebook=5.2.2=py27_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py27_0
+- pathlib2=2.3.0=py27_0
+- pexpect=4.3.1=py27_0
+- pickleshare=0.7.4=py27_0
+- pip=9.0.1=py27_1
+- prompt_toolkit=1.0.15=py27_0
+- ptyprocess=0.5.2=py27_0
+- pygments=2.2.0=py27_0
+- python=2.7.14=4
+- python-dateutil=2.6.1=py27_0
+- pyzmq=16.0.2=py27_2
+- readline=7.0=0
+- scandir=1.6=py27_0
+- setuptools=38.2.4=py27_0
+- simplegeneric=0.8.1=py27_0
+- singledispatch=3.4.0.3=py27_0
+- six=1.11.0=py27_1
+- sqlite=3.20.1=2
+- ssl_match_hostname=3.5.0.1=py27_1
+- subprocess32=3.2.7=py27_0
+- terminado=0.8.1=py27_0
+- testpath=0.3.1=py27_0
+- tk=8.6.7=0
+- tornado=4.5.2=py27_0
+- traitlets=4.3.2=py27_0
+- wcwidth=0.1.7=py27_0
+- webencodings=0.5=py27_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py27_0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

ADD
repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml
+# AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:31:22 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py35_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py35_0
+- decorator=4.1.2=py35_0
+- entrypoints=0.2.3=py35_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py35_0
+- ipython=6.2.1=py35_1
+- ipython_genutils=0.2.0=py35_0
+- ipywidgets=6.0.1=py35_0
+- jedi=0.10.2=py35_0
+- jinja2=2.10=py35_0
+- jsonschema=2.6.0=py35_0
+- jupyter_client=5.2.0=py35_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py35_0
+- jupyterlab_launcher=0.6.0=py35_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py35_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py35_0
+- ncurses=5.9=10
+- notebook=5.2.2=py35_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py35_0
+- pexpect=4.3.1=py35_0
+- pickleshare=0.7.4=py35_0
+- pip=9.0.1=py35_1
+- prompt_toolkit=1.0.15=py35_0
+- ptyprocess=0.5.2=py35_0
+- pygments=2.2.0=py35_0
+- python=3.5.4=3
+- python-dateutil=2.6.1=py35_0
+- pyzmq=16.0.2=py35_2
+- readline=7.0=0
+- setuptools=38.2.4=py35_0
+- simplegeneric=0.8.1=py35_0
+- six=1.11.0=py35_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py35_0
+- testpath=0.3.1=py35_0
+- tk=8.6.7=0
+- tornado=4.5.2=py35_0
+- traitlets=4.3.2=py35_0
+- wcwidth=0.1.7=py35_0
+- webencodings=0.5=py35_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py35_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

ADD
repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py36_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py36_0
+- decorator=4.1.2=py36_0
+- entrypoints=0.2.3=py36_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py36_0
+- ipython=6.2.1=py36_1
+- ipython_genutils=0.2.0=py36_0
+- ipywidgets=6.0.1=py36_0
+- jedi=0.10.2=py36_0
+- jinja2=2.10=py36_0
+- jsonschema=2.6.0=py36_0
+- jupyter_client=5.2.0=py36_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py36_0
+- jupyterlab_launcher=0.6.0=py36_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py36_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py36_0
+- ncurses=5.9=10
+- notebook=5.2.2=py36_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py36_0
+- pexpect=4.3.1=py36_0
+- pickleshare=0.7.4=py36_0
+- pip=9.0.1=py36_1
+- prompt_toolkit=1.0.15=py36_0
+- ptyprocess=0.5.2=py36_0
+- pygments=2.2.0=py36_0
+- python=3.6.4=0
+- python-dateutil=2.6.1=py36_0
+- pyzmq=16.0.2=py36_2
+- readline=7.0=0
+- setuptools=38.2.4=py36_0
+- simplegeneric=0.8.1=py36_0
+- six=1.11.0=py36_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py36_0
+- testpath=0.3.1=py36_0
+- tk=8.6.7=0
+- tornado=4.5.2=py36_0
+- traitlets=4.3.2=py36_0
+- wcwidth=0.1.7=py36_0
+- webencodings=0.5=py36_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py36_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

MODIFY
repo2docker/buildpacks/conda/environment.yml
repo2docker/buildpacks/conda/environment.yml
dependencies:
- - python==3.6.1
+ - python==3.6.*
- ipywidgets==6.0.1
- jupyterlab==0.30.6
- notebook==5.2.2

MODIFY
repo2docker/buildpacks/conda/freeze.py
repo2docker/buildpacks/conda/freeze.py
@@ -8,6 +8,7 @@ Run in a continuumio/miniconda3 image to ensure portability
from datetime import datetime
import os
import pathlib
+import shutil
from subprocess import check_call
from ruamel.yaml import YAML
@@ -18,7 +19,10 @@ MINICONDA_VERSION = '4.3.27'
HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
ENV_FILE = 'environment.yml'
-FROZEN_FILE = 'environment.frozen.yml'
+FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
+
+ENV_FILE_T = 'environment.py-{py}.yml'
+FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
yaml = YAML(typ='rt')
@@ -78,11 +82,40 @@ def freeze(env_file, frozen_file):
'conda config --add channels conda-forge',
'conda config --system --set auto_update_conda false',
f"conda env create -v -f /r2d/{env_file} -n r2d",
- f"conda env export -n r2d > /r2d/{frozen_file}",
+ f"conda env export -n r2d >> /r2d/{frozen_file}",
])
])
fixup(HERE / frozen_file)
+def set_python(py_env_file, py):
+ """Set the Python version in an env file"""
+ if os.path.exists(py_env_file):
+ # only clobber auto-generated files
+ with open(py_env_file) as f:
+ text = f.read()
+ if text and 'GENERATED' not in text:
+ return
+ with open(ENV_FILE) as f:
+ env = yaml.load(f)
+ for idx, dep in enumerate(env['dependencies']):
+ if dep.split('=')[0] == 'python':
+ env['dependencies'][idx] = f'python={py}.*'
+ break
+ else:
+ raise ValueError(f"python dependency not found in {env['dependencies']}")
+ # update python dependency
+ with open(py_env_file, 'w') as f:
+ f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
+ f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
+ yaml.dump(env, f)
+
+
if __name__ == '__main__':
- freeze(ENV_FILE, FROZEN_FILE)
+ for py in ('2.7', '3.5', '3.6'):
+ env_file = ENV_FILE_T.format(py=py)
+ set_python(env_file, py)
+ frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
+ freeze(env_file, frozen_file)
+ # use last version as default
+ shutil.copy(frozen_file, FROZEN_FILE)

[conda] create separate frozen envs for Python versions

this will allow us to create different base envs based on Python major.minor versions