Split detectors.py into individual files
These were buildpacks now anyway, not detectors.
MODIFY
repo2docker/app.py
repo2docker/app.py
@@ -25,7 +25,7 @@ from docker.utils import kwargs_from_env
import subprocess
-from .detectors import (
+from .buildpacks import (
PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage
)

ADD
repo2docker/buildpacks/__init__.py
+from .base import BuildPack, BaseImage
+from .python import Python2BuildPack, PythonBuildPack
+from .conda import CondaBuildPack
+from .julia import JuliaBuildPack
+from .docker import DockerBuildPack
+from .legacy import LegacyBinderDockerBuildPack

RENAME
repo2docker/detectors.py
repo2docker/buildpacks/base.py
-"""
-Generates a variety of Dockerfiles based on an input matrix
-"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
@@ -385,7 +382,7 @@ class BuildPack(LoggingConfigurable):
for src in sorted(self.build_script_files):
src_parts = src.split('/')
- src_path = os.path.join(os.path.dirname(__file__), 'files', *src_parts)
+ src_path = os.path.join(os.path.dirname(__file__), '..', 'files', *src_parts)
tar.add(src_path, src, filter=_filter_tar)
tar.add('.', 'src/', filter=_filter_tar)
@@ -453,315 +450,3 @@ class BaseImage(BuildPack):
post_build, DOC_URL+'#system-post-build-scripts'))
return [post_build]
return []
-
-class PythonBuildPack(BuildPack):
- name = "python3.5"
- version = "0.1"
-
- packages = {
- 'python3',
- 'python3-venv',
- 'python3-dev',
- }
-
- env = [
- ("VENV_PATH", "${APP_BASE}/venv"),
- # Prefix to use for installing kernels and finding jupyter binary
- ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
- ]
-
- path = [
- "${VENV_PATH}/bin"
- ]
-
- build_scripts = [
- (
- "root",
- r"""
- mkdir -p ${VENV_PATH} && \
- chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- python3 -m venv ${VENV_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- pip install --no-cache-dir \
- notebook==5.2.2 \
- ipywidgets==6.0.0 \
- jupyterlab==0.28 && \
- jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
- jupyter serverextension enable --py jupyterlab --sys-prefix
- """
- )
- ]
-
- @default('assemble_scripts')
- def setup_assembly(self):
- # If we have a runtime.txt & that's set to python-2.7,
- # we will *not* install requirements.txt but will find &
- # install a requirements3.txt file if it exists.
- # This way, when using python2 venv, requirements.txt will
- # be installed in the python2 venv, and requirements3.txt
- # will be installed in python3 venv. This is less of a
- # surprise than requiring python2 to be requirements2.txt tho.
- try:
- with open(self.binder_path('runtime.txt')) as f:
- runtime = f.read().strip()
- except FileNotFoundError:
- runtime = 'python-3.5'
- if runtime == 'python-2.7':
- requirements_file = self.binder_path('requirements3.txt')
- else:
- requirements_file = self.binder_path('requirements.txt')
- if os.path.exists(requirements_file):
- return [(
- '${NB_USER}',
- 'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
- )]
- return []
-
- def detect(self):
- return os.path.exists('requirements.txt') and super().detect()
-
-class CondaBuildPack(BuildPack):
- name = "conda"
- version = "0.1"
- env = [
- ('CONDA_DIR', '${APP_BASE}/conda'),
- ('NB_PYTHON_PREFIX', '${CONDA_DIR}')
- ]
-
- path = ['${CONDA_DIR}/bin']
-
- build_script_files = {
- 'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
- 'conda/environment.yml': '/tmp/environment.yml'
- }
-
- build_scripts = [
- (
- "root",
- r"""
- bash /tmp/install-miniconda.bash && \
- rm /tmp/install-miniconda.bash /tmp/environment.yml
- """
- )
- ]
-
- @default('assemble_scripts')
- def setup_assembly(self):
- assembly_scripts = []
- environment_yml = self.binder_path('environment.yml')
- if os.path.exists(environment_yml):
- assembly_scripts.append((
- '${NB_USER}',
- r"""
- conda env update -v -n root -f "{}" && \
- conda clean -tipsy
- """.format(environment_yml)
- ))
- return assembly_scripts
-
- def detect(self):
- return os.path.exists(self.binder_path('environment.yml')) and super().detect()
-
-
-class Python2BuildPack(BuildPack):
- name = "python2.7"
- version = "0.1"
-
- packages = {
- 'python',
- 'python-dev',
- 'virtualenv'
- }
-
- env = [
- ('VENV2_PATH', '${APP_BASE}/venv2')
- ]
-
- path = [
- "${VENV2_PATH}/bin"
- ]
-
- build_scripts = [
- (
- "root",
- r"""
- mkdir -p ${VENV2_PATH} && \
- chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- virtualenv -p python2 ${VENV2_PATH}
- """
- ),
- (
- "${NB_USER}",
- r"""
- pip2 install --no-cache-dir \
- ipykernel==4.6.1 && \
- python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
- """
- )
- ]
-
- @default('assemble_scripts')
- def setup_assembly(self):
- return [
- (
- '${NB_USER}',
- 'pip2 install --no-cache-dir -r requirements.txt'
- )
- ]
-
- def detect(self):
- requirements_txt = self.binder_path('requirements.txt')
- runtime_txt = self.binder_path('runtime.txt')
- if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
- with open(runtime_txt) as f:
- runtime = f.read().strip()
- if runtime == 'python-2.7':
- return True
- return False
-
-class JuliaBuildPack(BuildPack):
- name = "julia"
- version = "0.1"
- env = [
- ('JULIA_PATH', '${APP_BASE}/julia'),
- ('JULIA_HOME', '${JULIA_PATH}/bin'),
- ('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
- ('JULIA_VERSION', '0.6.0'),
- ('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
- ]
-
- path = [
- '${JULIA_PATH}/bin'
- ]
-
- build_scripts = [
- (
- "root",
- r"""
- mkdir -p ${JULIA_PATH} && \
- curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
- """
- ),
- (
- "root",
- r"""
- mkdir -p ${JULIA_PKGDIR} && \
- chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
- """
- ),
- (
- "${NB_USER}",
- # HACK: Can't seem to tell IJulia to install in sys-prefix
- # FIXME: Find way to get it to install under /srv and not $HOME?
- r"""
- julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
- mv ${HOME}/.local/share/jupyter/kernels/julia-0.6 ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
- """
- )
- ]
-
- @default('assemble_scripts')
- def setup_assembly(self):
- require = self.binder_path('REQUIRE')
- return [(
- "${NB_USER}",
- # Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
- # right thing
- r"""
- cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
- julia -e ' \
- Pkg.resolve(); \
- for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
- pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
- end \
- '
- """ % { "require" : require }
- )]
-
- def detect(self):
- return os.path.exists(self.binder_path('REQUIRE')) and super()
-
-
-class DockerBuildPack(BuildPack):
- name = "Dockerfile"
- dockerfile = "Dockerfile"
-
- def detect(self):
- return os.path.exists(self.binder_path('Dockerfile'))
-
- def render(self):
- Dockerfile = self.binder_path('Dockerfile')
- with open(Dockerfile) as f:
- return f.read()
-
- def build(self, image_spec):
- client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
- for line in client.build(
- path=os.getcwd(),
- dockerfile=self.binder_path(self.dockerfile),
- tag=image_spec,
- buildargs={},
- decode=True,
- forcerm=True,
- rm=True
- ):
- yield line
-
-class LegacyBinderDockerBuildPack(DockerBuildPack):
-
- name = 'Legacy Binder Dockerfile'
- dockerfile = '._binder.Dockerfile'
-
- dockerfile_appendix = Unicode(dedent(r"""
- USER root
- COPY . /home/main/notebooks
- RUN chown -R main:main /home/main/notebooks
- USER main
- WORKDIR /home/main/notebooks
- ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
- RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
- conda remove -yq -n python3 nb_conda_kernels && \
- conda install -yq -n root ipykernel==4.6.0 && \
- /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
- /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
- /home/main/anaconda2/bin/ipython kernel install --sys-prefix
- ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
- CMD jupyter notebook --ip 0.0.0.0
- """), config=True)
-
- def render(self):
- with open('Dockerfile') as f:
- return f.read() + self.dockerfile_appendix
-
- def build(self, image_spec):
- with open(self.dockerfile, 'w') as f:
- f.write(self.render())
- return super().build(image_spec)
-
- def detect(self):
- try:
- with open('Dockerfile', 'r') as f:
- for line in f:
- if line.startswith('FROM'):
- if 'andrewosh/binder-base' in line.split('#')[0].lower():
- return True
- else:
- return False
- except FileNotFoundError:
- pass
-
- return False

ADD
repo2docker/buildpacks/conda.py
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class CondaBuildPack(BuildPack):
+ name = "conda"
+ version = "0.1"
+ env = [
+ ('CONDA_DIR', '${APP_BASE}/conda'),
+ ('NB_PYTHON_PREFIX', '${CONDA_DIR}')
+ ]
+
+ path = ['${CONDA_DIR}/bin']
+
+ build_script_files = {
+ 'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
+ 'conda/environment.yml': '/tmp/environment.yml'
+ }
+
+ build_scripts = [
+ (
+ "root",
+ r"""
+ bash /tmp/install-miniconda.bash && \
+ rm /tmp/install-miniconda.bash /tmp/environment.yml
+ """
+ )
+ ]
+
+ @default('assemble_scripts')
+ def setup_assembly(self):
+ assembly_scripts = []
+ environment_yml = self.binder_path('environment.yml')
+ if os.path.exists(environment_yml):
+ assembly_scripts.append((
+ '${NB_USER}',
+ r"""
+ conda env update -v -n root -f "{}" && \
+ conda clean -tipsy
+ """.format(environment_yml)
+ ))
+ return assembly_scripts
+
+ def detect(self):
+ return os.path.exists(self.binder_path('environment.yml')) and super().detect()

ADD
repo2docker/buildpacks/docker.py
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class DockerBuildPack(BuildPack):
+ name = "Dockerfile"
+ dockerfile = "Dockerfile"
+
+ def detect(self):
+ return os.path.exists(self.binder_path('Dockerfile'))
+
+ def render(self):
+ Dockerfile = self.binder_path('Dockerfile')
+ with open(Dockerfile) as f:
+ return f.read()
+
+ def build(self, image_spec):
+ client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
+ for line in client.build(
+ path=os.getcwd(),
+ dockerfile=self.binder_path(self.dockerfile),
+ tag=image_spec,
+ buildargs={},
+ decode=True,
+ forcerm=True,
+ rm=True
+ ):
+ yield line

ADD
repo2docker/buildpacks/julia.py
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class JuliaBuildPack(BuildPack):
+ name = "julia"
+ version = "0.1"
+ env = [
+ ('JULIA_PATH', '${APP_BASE}/julia'),
+ ('JULIA_HOME', '${JULIA_PATH}/bin'),
+ ('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
+ ('JULIA_VERSION', '0.6.0'),
+ ('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
+ ]
+
+ path = [
+ '${JULIA_PATH}/bin'
+ ]
+
+ build_scripts = [
+ (
+ "root",
+ r"""
+ mkdir -p ${JULIA_PATH} && \
+ curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
+ """
+ ),
+ (
+ "root",
+ r"""
+ mkdir -p ${JULIA_PKGDIR} && \
+ chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
+ """
+ ),
+ (
+ "${NB_USER}",
+ # HACK: Can't seem to tell IJulia to install in sys-prefix
+ # FIXME: Find way to get it to install under /srv and not $HOME?
+ r"""
+ julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
+ mv ${HOME}/.local/share/jupyter/kernels/julia-0.6 ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
+ """
+ )
+ ]
+
+ @default('assemble_scripts')
+ def setup_assembly(self):
+ require = self.binder_path('REQUIRE')
+ return [(
+ "${NB_USER}",
+ # Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
+ # right thing
+ r"""
+ cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
+ julia -e ' \
+ Pkg.resolve(); \
+ for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
+ pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
+ end \
+ '
+ """ % { "require" : require }
+ )]
+
+ def detect(self):
+ return os.path.exists(self.binder_path('REQUIRE')) and super()

ADD
repo2docker/buildpacks/legacy.py
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .docker import DockerBuildPack
+
+class LegacyBinderDockerBuildPack(DockerBuildPack):
+
+ name = 'Legacy Binder Dockerfile'
+ dockerfile = '._binder.Dockerfile'
+
+ dockerfile_appendix = Unicode(dedent(r"""
+ USER root
+ COPY . /home/main/notebooks
+ RUN chown -R main:main /home/main/notebooks
+ USER main
+ WORKDIR /home/main/notebooks
+ ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
+ RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
+ conda remove -yq -n python3 nb_conda_kernels && \
+ conda install -yq -n root ipykernel==4.6.0 && \
+ /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
+ /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
+ /home/main/anaconda2/bin/ipython kernel install --sys-prefix
+ ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
+ CMD jupyter notebook --ip 0.0.0.0
+ """), config=True)
+
+ def render(self):
+ with open('Dockerfile') as f:
+ return f.read() + self.dockerfile_appendix
+
+ def build(self, image_spec):
+ with open(self.dockerfile, 'w') as f:
+ f.write(self.render())
+ return super().build(image_spec)
+
+ def detect(self):
+ try:
+ with open('Dockerfile', 'r') as f:
+ for line in f:
+ if line.startswith('FROM'):
+ if 'andrewosh/binder-base' in line.split('#')[0].lower():
+ return True
+ else:
+ return False
+ except FileNotFoundError:
+ pass
+
+ return False

ADD
repo2docker/buildpacks/python.py
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class PythonBuildPack(BuildPack):
+ name = "python3.5"
+ version = "0.1"
+
+ packages = {
+ 'python3',
+ 'python3-venv',
+ 'python3-dev',
+ }
+
+ env = [
+ ("VENV_PATH", "${APP_BASE}/venv"),
+ # Prefix to use for installing kernels and finding jupyter binary
+ ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
+ ]
+
+ path = [
+ "${VENV_PATH}/bin"
+ ]
+
+ build_scripts = [
+ (
+ "root",
+ r"""
+ mkdir -p ${VENV_PATH} && \
+ chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
+ """
+ ),
+ (
+ "${NB_USER}",
+ r"""
+ python3 -m venv ${VENV_PATH}
+ """
+ ),
+ (
+ "${NB_USER}",
+ r"""
+ pip install --no-cache-dir \
+ notebook==5.2.2 \
+ ipywidgets==6.0.0 \
+ jupyterlab==0.28 && \
+ jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
+ jupyter serverextension enable --py jupyterlab --sys-prefix
+ """
+ )
+ ]
+
+ @default('assemble_scripts')
+ def setup_assembly(self):
+ # If we have a runtime.txt & that's set to python-2.7,
+ # we will *not* install requirements.txt but will find &
+ # install a requirements3.txt file if it exists.
+ # This way, when using python2 venv, requirements.txt will
+ # be installed in the python2 venv, and requirements3.txt
+ # will be installed in python3 venv. This is less of a
+ # surprise than requiring python2 to be requirements2.txt tho.
+ try:
+ with open(self.binder_path('runtime.txt')) as f:
+ runtime = f.read().strip()
+ except FileNotFoundError:
+ runtime = 'python-3.5'
+ if runtime == 'python-2.7':
+ requirements_file = self.binder_path('requirements3.txt')
+ else:
+ requirements_file = self.binder_path('requirements.txt')
+ if os.path.exists(requirements_file):
+ return [(
+ '${NB_USER}',
+ 'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
+ )]
+ return []
+
+ def detect(self):
+ return os.path.exists('requirements.txt') and super().detect()
+
+
+class Python2BuildPack(BuildPack):
+ name = "python2.7"
+ version = "0.1"
+
+ packages = {
+ 'python',
+ 'python-dev',
+ 'virtualenv'
+ }
+
+ env = [
+ ('VENV2_PATH', '${APP_BASE}/venv2')
+ ]
+
+ path = [
+ "${VENV2_PATH}/bin"
+ ]
+
+ build_scripts = [
+ (
+ "root",
+ r"""
+ mkdir -p ${VENV2_PATH} && \
+ chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
+ """
+ ),
+ (
+ "${NB_USER}",
+ r"""
+ virtualenv -p python2 ${VENV2_PATH}
+ """
+ ),
+ (
+ "${NB_USER}",
+ r"""
+ pip2 install --no-cache-dir \
+ ipykernel==4.6.1 && \
+ python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
+ """
+ )
+ ]
+
+ @default('assemble_scripts')
+ def setup_assembly(self):
+ return [
+ (
+ '${NB_USER}',
+ 'pip2 install --no-cache-dir -r requirements.txt'
+ )
+ ]
+
+ def detect(self):
+ requirements_txt = self.binder_path('requirements.txt')
+ runtime_txt = self.binder_path('runtime.txt')
+ if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
+ with open(runtime_txt) as f:
+ runtime = f.read().strip()
+ if runtime == 'python-2.7':
+ return True
+ return False

Split detectors.py into individual files

These were buildpacks now anyway, not detectors.