Start reusing existing docker images if content hasn't changed
Content providers can specify a "content ID" which is used to identify
versions of the content. The ID is used in the docker image name and if
we find an existing image for a given content ID the build step is
skipped.
MODIFY
repo2docker/app.py
repo2docker/app.py
@@ -13,7 +13,6 @@ import sys
import logging
import os
import pwd
-import subprocess
import tempfile
import time
@@ -215,6 +214,15 @@ class Repo2Docker(Application):
spec, checkout_path, yield_output=self.json_logs):
self.log.info(log_line, extra=dict(phase='fetching'))
+ self.output_image_spec = (
+ 'r2d' +
+ escapism.escape(self.repo, escape_char='-').lower()
+ )
+ if picked_content_provider.content_id is not None:
+ self.output_image_spec += picked_content_provider.content_id
+ else:
+ self.output_image_spec += str(int(time.time()))
+
def validate_image_name(self, image_name):
"""
Validate image_name read by argparse
@@ -471,13 +479,8 @@ class Repo2Docker(Application):
if args.image_name:
self.output_image_spec = args.image_name
else:
- # Attempt to set a sane default!
- # HACK: Provide something more descriptive?
- self.output_image_spec = (
- 'r2d' +
- escapism.escape(self.repo, escape_char='-').lower() +
- str(int(time.time()))
- )
+ # we will pick a name after fetching the repository
+ self.output_image_spec = None
self.push = args.push
self.run = args.run
@@ -669,40 +672,18 @@ class Repo2Docker(Application):
s.close()
return port
- def start(self):
- """Start execution of repo2docker"""
- # Check if r2d can connect to docker daemon
- if self.build:
- try:
- client = docker.APIClient(version='auto',
- **kwargs_from_env())
- del client
- except DockerException as e:
- print("Docker client initialization error. Check if docker is"
- " running on the host.")
- print(e)
- if self.log_level == logging.DEBUG:
- raise e
- sys.exit(1)
-
- # If the source to be executed is a directory, continue using the
- # directory. In the case of a local directory, it is used as both the
- # source and target. Reusing a local directory seems better than
- # making a copy of it as it might contain large files that would be
- # expensive to copy.
- if os.path.isdir(self.repo):
- checkout_path = self.repo
- else:
- if self.git_workdir is None:
- checkout_path = tempfile.mkdtemp(prefix='repo2docker')
- else:
- checkout_path = self.git_workdir
+ def find_image(self):
+ # check if we already have an image for this content
+ client = docker.APIClient(version='auto', **kwargs_from_env())
+ for image in client.images():
+ if image['RepoTags'] is not None:
+ for tags in image['RepoTags']:
+ if tags == self.output_image_spec + ":latest":
+ return True
- # keep as much as possible in the context manager to make sure we
- # cleanup if things go wrong
- with maybe_cleanup(checkout_path, self.cleanup_checkout):
- self.fetch(self.repo, self.ref, checkout_path)
+ return False
+ def _build_image(self, checkout_path):
if self.subdir:
checkout_path = os.path.join(checkout_path, self.subdir)
if not os.path.isdir(checkout_path):
@@ -747,6 +728,45 @@ class Repo2Docker(Application):
self.log.info(json.dumps(l),
extra=dict(phase='building'))
+ def start(self):
+ """Start execution of repo2docker"""
+ # Check if r2d can connect to docker daemon
+ if self.build:
+ try:
+ docker.APIClient(version='auto', **kwargs_from_env())
+ except DockerException as e:
+ print("Docker client initialization error. Check if docker is"
+ " running on the host.")
+ print(e)
+ if self.log_level == logging.DEBUG:
+ raise e
+ sys.exit(1)
+
+ # If the source to be executed is a directory, continue using the
+ # directory. In the case of a local directory, it is used as both the
+ # source and target. Reusing a local directory seems better than
+ # making a copy of it as it might contain large files that would be
+ # expensive to copy.
+ if os.path.isdir(self.repo):
+ checkout_path = self.repo
+ else:
+ if self.git_workdir is None:
+ checkout_path = tempfile.mkdtemp(prefix='repo2docker')
+ else:
+ checkout_path = self.git_workdir
+
+ # keep as much as possible in the context manager to make sure we
+ # cleanup if things go wrong
+ with maybe_cleanup(checkout_path, self.cleanup_checkout):
+ self.fetch(self.repo, self.ref, checkout_path)
+
+ if self.find_image():
+ self.log.info("Reusing existing image ({}), not "
+ "building.".format(self.output_image_spec))
+
+ else:
+ self._build_image(checkout_path)
+
if self.push:
self.push_image()

MODIFY
repo2docker/contentproviders/base.py
repo2docker/contentproviders/base.py
@@ -17,6 +17,24 @@ class ContentProvider:
def __init__(self):
self.log = logging.getLogger("repo2docker")
+ @property
+ def content_id(self):
+ """A unique ID to represent the version of the content.
+
+ This ID is used to name the built images. If the ID is the same between
+ two runs of repo2docker we will reuse an existing image (if it exists).
+
+ By providing an ID that summarizes the content we can reuse existing
+ images and speed up build times. A good ID is the revision of a Git
+ repository or a hash computed from all the content.
+
+ The type content ID can be any string.
+
+ To disable this behaviour set this property to `None` in which case
+ a fresh image will always be built.
+ """
+ return None
+
def detect(self, repo, ref=None, extra_args=None):
"""Determine compatibility between source and this provider.

MODIFY
repo2docker/contentproviders/git.py
repo2docker/contentproviders/git.py
@@ -44,3 +44,15 @@ class Git(ContentProvider):
cwd=output_dir,
capture=yield_output):
yield line
+
+ cmd = ['git', 'rev-parse', 'HEAD']
+ sha1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=output_dir)
+ self._sha1 = sha1.stdout.read().decode().strip()
+
+ @property
+ def content_id(self):
+ """A unique ID to represent the version of the content.
+
+ Uses the git commit ID of the repository.
+ """
+ return self._sha1

Start reusing existing docker images if content hasn't changed

Content providers can specify a "content ID" which is used to identify
versions of the content. The ID is used in the docker image name and if
we find an existing image for a given content ID the build step is
skipped.