check for local pip requirements in conda environments
MODIFY
repo2docker/buildpacks/conda/__init__.py
repo2docker/buildpacks/conda/__init__.py
@@ -6,6 +6,7 @@ from collections import Mapping
from ruamel.yaml import YAML
from ..base import BaseImage
+from ...utils import is_local_pip_requirement
# pattern for parsing conda dependency line
PYTHON_REGEX = re.compile(r"python\s*=+\s*([\d\.]*)")
@@ -127,20 +128,18 @@ class CondaBuildPack(BaseImage):
files.update(super().get_build_script_files())
return files
- @property
- def python_version(self):
- """Detect the Python version for a given `environment.yml`
+ _environment_yaml = None
- Will return 'x.y' if version is found (e.g '3.6'),
- or a Falsy empty string '' if not found.
+ @property
+ def environment_yaml(self):
+ if self._environment_yaml is not None:
+ return self._environment_yaml
- """
environment_yml = self.binder_path("environment.yml")
if not os.path.exists(environment_yml):
- return ""
+ self._environment_yaml = {}
+ return self._environment_yaml
- if not hasattr(self, "_python_version"):
- py_version = None
with open(environment_yml) as f:
env = YAML().load(f)
# check if the env file is empty, if so instantiate an empty dictionary.
@@ -149,9 +148,41 @@ class CondaBuildPack(BaseImage):
# check if the env file provided a dick-like thing not a list or other data structure.
if not isinstance(env, Mapping):
raise TypeError(
- "environment.yml should contain a dictionary. Got %r"
- % type(env)
+ "environment.yml should contain a dictionary. Got %r" % type(env)
)
+ self._environment_yaml = env
+
+ return self._environment_yaml
+
+ @property
+ def _should_preassemble_env(self):
+ """Check for local pip requirements in environment.yaml
+
+ If there are any local references, e.g. `-e .`,
+ stage the whole repo prior to installation.
+ """
+ dependencies = self.environment_yaml.get("dependencies", [])
+ pip_requirements = None
+ for dep in dependencies:
+ if isinstance(dep, dict) and dep.get("pip"):
+ pip_requirements = dep["pip"]
+ if isinstance(pip_requirements, list):
+ for line in pip_requirements:
+ if is_local_pip_requirement(line):
+ return False
+ return True
+
+ @property
+ def python_version(self):
+ """Detect the Python version for a given `environment.yml`
+
+ Will return 'x.y' if version is found (e.g '3.6'),
+ or a Falsy empty string '' if not found.
+
+ """
+ if not hasattr(self, "_python_version"):
+ py_version = None
+ env = self.environment_yaml
for dep in env.get("dependencies", []):
if not isinstance(dep, str):
continue
@@ -185,19 +216,20 @@ class CondaBuildPack(BaseImage):
repo contents change
"""
assemble_files = super().get_preassemble_script_files()
+ if self._should_preassemble_env:
environment_yml = self.binder_path("environment.yml")
if os.path.exists(environment_yml):
assemble_files[environment_yml] = environment_yml
return assemble_files
- def get_preassemble_scripts(self):
+ def get_env_scripts(self):
"""Return series of build-steps specific to this source repository.
"""
- assembly_scripts = []
+ scripts = []
environment_yml = self.binder_path("environment.yml")
env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}"
if os.path.exists(environment_yml):
- assembly_scripts.append(
+ scripts.append(
(
"${NB_USER}",
r"""
@@ -209,7 +241,19 @@ class CondaBuildPack(BaseImage):
),
)
)
- return super().get_preassemble_scripts() + assembly_scripts
+ return scripts
+
+ def get_preassemble_scripts(self):
+ scripts = super().get_preassemble_scripts()
+ if self._should_preassemble_env:
+ scripts.extend(self.get_env_scripts())
+ return scripts
+
+ def get_assemble_scripts(self):
+ scripts = super().get_assemble_scripts()
+ if not self._should_preassemble_env:
+ scripts.extend(self.get_env_scripts())
+ return scripts
def detect(self):
"""Check if current repo should be built with the Conda BuildPack.

MODIFY
repo2docker/buildpacks/python/__init__.py
repo2docker/buildpacks/python/__init__.py
import os
from ..conda import CondaBuildPack
+from ...utils import is_local_pip_requirement
class PythonBuildPack(CondaBuildPack):
@@ -34,31 +35,6 @@ class PythonBuildPack(CondaBuildPack):
self._python_version = py_version
return self._python_version
- def _is_local_requirement(self, line):
- """Return whether a line in a requirements.txt file references a local file"""
- # trim comments and skip empty lines
- line = line.split("#", 1)[0].strip()
- if not line:
- return False
- if line.startswith(("-r", "-c")):
- # local -r or -c references break isolation
- return True
- # strip off `-e, etc.`
- if line.startswith("-"):
- line = line.split(None, 1)[1]
- if "file://" in line:
- # file references break isolation
- return True
- if "://" in line:
- # handle git://../local/file
- path = line.split("://", 1)[1]
- else:
- path = line
- if path.startswith("."):
- # references a local file
- return True
- return False
-
def _get_pip_scripts(self):
"""Get pip install scripts
@@ -112,7 +88,7 @@ class PythonBuildPack(CondaBuildPack):
continue
with open(requirements_txt) as f:
for line in f:
- if self._is_local_requirement(line):
+ if is_local_pip_requirement(line):
return False
# didn't find any local references,

MODIFY
repo2docker/utils.py
repo2docker/utils.py
@@ -431,3 +431,29 @@ def normalize_doi(val):
(e.g. https://doi.org/10.1234/jshd123)"""
m = doi_regexp.match(val)
return m.group(2)
+
+
+def is_local_pip_requirement(line):
+ """Return whether a pip requirement (e.g. in requirements.txt file) references a local file"""
+ # trim comments and skip empty lines
+ line = line.split("#", 1)[0].strip()
+ if not line:
+ return False
+ if line.startswith(("-r", "-c")):
+ # local -r or -c references break isolation
+ return True
+ # strip off `-e, etc.`
+ if line.startswith("-"):
+ line = line.split(None, 1)[1]
+ if "file://" in line:
+ # file references break isolation
+ return True
+ if "://" in line:
+ # handle git://../local/file
+ path = line.split("://", 1)[1]
+ else:
+ path = line
+ if path.startswith("."):
+ # references a local file
+ return True
+ return False

check for local pip requirements in conda environments