preassemble python
MODIFY
repo2docker/buildpacks/python/__init__.py
repo2docker/buildpacks/python/__init__.py
@@ -34,50 +34,79 @@ class PythonBuildPack(CondaBuildPack):
self._python_version = py_version
return self._python_version
- def get_assemble_files(self):
- assemble_files = super().get_assemble_files()
- for name in ('requirements.txt', 'requirements3.txt'):
- requirements_txt = self.binder_path(name)
- if os.path.exists(requirements_txt):
- assemble_files.append(requirements_txt)
- return assemble_files
-
def _is_local_requirement(self, line):
"""Return whether a line in a requirements.txt file references a local file"""
# trim comments and skip empty lines
- line = line.split('#', 1)[0].strip()
+ line = line.split("#", 1)[0].strip()
if not line:
return False
- if line.startswith(('-r', '-c')):
+ if line.startswith(("-r", "-c")):
# local -r or -c references break isolation
return True
# strip off `-e, etc.`
- if line.startswith('-'):
+ if line.startswith("-"):
line = line.split(None, 1)[1]
- if 'file://' in line:
+ if "file://" in line:
# file references break isolation
return True
- if '://' in line:
+ if "://" in line:
# handle git://../local/file
- path = line.split('://', 1)[1]
+ path = line.split("://", 1)[1]
else:
path = line
- if path.startswith('.'):
+ if path.startswith("."):
# references a local file
return True
return False
+ def _get_pip_scripts(self):
+ """Get pip install scripts
+
+ added to preassemble unless local references are found,
+ in which case this happens in assemble.
+ """
+ # KERNEL_PYTHON_PREFIX is the env with the kernel,
+ # whether it's distinct from the notebook or the same.
+ pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
+ scripts = []
+ if self.py2:
+ # using python 2 kernel,
+ # requirements3.txt allows installation in the notebook server env
+ nb_requirements_file = self.binder_path("requirements3.txt")
+ if os.path.exists(nb_requirements_file):
+ scripts.append(
+ (
+ "${NB_USER}",
+ # want the $NB_PYHTON_PREFIX environment variable, not for
+ # Python's string formatting to try and replace this
+ '${{NB_PYTHON_PREFIX}}/bin/pip install --no-cache-dir -r "{}"'.format(
+ nb_requirements_file
+ ),
+ )
+ )
+
+ # install requirements.txt in the kernel env
+ requirements_file = self.binder_path("requirements.txt")
+ if os.path.exists(requirements_file):
+ scripts.append(
+ (
+ "${NB_USER}",
+ '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file),
+ )
+ )
+ return scripts
+
@property
- def assemble_from_subset(self):
+ def _should_preassemble_pip(self):
"""Peek in requirements.txt to determine if we can assemble from only env files
If there are any local references, e.g. `-e .`,
stage the whole repo prior to installation.
"""
- if not os.path.exists('binder') and os.path.exists('setup.py'):
+ if not os.path.exists("binder") and os.path.exists("setup.py"):
# can't install from subset if we're using setup.py
return False
- for name in ('requirements.txt', 'requirements3.txt'):
+ for name in ("requirements.txt", "requirements3.txt"):
requirements_txt = self.binder_path(name)
if not os.path.exists(requirements_txt):
continue
@@ -90,9 +119,23 @@ class PythonBuildPack(CondaBuildPack):
# allow assembly from subset
return True
+ def get_preassemble_script_files(self):
+ assemble_files = super().get_preassemble_script_files()
+ for name in ("requirements.txt", "requirements3.txt"):
+ requirements_txt = self.binder_path(name)
+ if os.path.exists(requirements_txt):
+ assemble_files[requirements_txt] = requirements_txt
+ return assemble_files
+
+ def get_preassemble_scripts(self):
+ """Return scripts to run before adding the full repository"""
+ scripts = super().get_preassemble_scripts()
+ if self._should_preassemble_pip:
+ scripts.extend(self._get_pip_scripts())
+ return scripts
+
def get_assemble_scripts(self):
- """Return series of build-steps specific to this repository.
- """
+ """Return series of build steps that require the full repository"""
# If we have a runtime.txt & that's set to python-2.7,
# requirements.txt will be installed in the *kernel* env
# and requirements3.txt (if it exists)
@@ -102,31 +145,8 @@ class PythonBuildPack(CondaBuildPack):
# KERNEL_PYTHON_PREFIX is the env with the kernel,
# whether it's distinct from the notebook or the same.
pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
- if self.py2:
- # using python 2 kernel,
- # requirements3.txt allows installation in the notebook server env
- nb_requirements_file = self.binder_path("requirements3.txt")
- if os.path.exists(nb_requirements_file):
- assemble_scripts.append(
- (
- "${NB_USER}",
- # want the $NB_PYHTON_PREFIX environment variable, not for
- # Python's string formatting to try and replace this
- '${{NB_PYTHON_PREFIX}}/bin/pip install --no-cache-dir -r "{}"'.format(
- nb_requirements_file
- ),
- )
- )
-
- # install requirements.txt in the kernel env
- requirements_file = self.binder_path("requirements.txt")
- if os.path.exists(requirements_file):
- assemble_scripts.append(
- (
- "${NB_USER}",
- '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file),
- )
- )
+ if not self._should_preassemble_pip:
+ assemble_scripts.extend(self._get_pip_scripts())
# setup.py exists *and* binder dir is not used
if not self.binder_dir and os.path.exists(setup_py):

preassemble python