text: #
  # BitBake Toaster Implementation
  #
  # Copyright (C) 2016        Intel Corporation
  #
  # This program is free software; you can redistribute it and/or modify
  # it under the terms of the GNU General Public License version 2 as
  # published by the Free Software Foundation.
  #
  # This program is distributed in the hope that it will be useful,
  # but WITHOUT ANY WARRANTY; without even the implied warranty of
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  # GNU General Public License for more details.
  #
  # You should have received a copy of the GNU General Public License along
  # with this program; if not, write to the Free Software Foundation, Inc.,
  # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  
  # Please run flake8 on this file before sending patches
  
  import os
  import re
  import logging
  import json
  import subprocess
  from collections import Counter
  from shutil import copyfile
  
  from orm.models import Project, ProjectTarget, Build, Layer_Version
  from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
  from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
  from orm.models import Layer, Target, Package, Package_Dependency
  from orm.models import ProjectVariable
  from bldcontrol.models import BuildRequest, BuildEnvironment
  from bldcontrol import bbcontroller
  
  from django.http import HttpResponse, JsonResponse
  from django.views.generic import View
  from django.core.urlresolvers import reverse
  from django.db.models import Q, F
  from django.db import Error
  from toastergui.templatetags.projecttags import filtered_filesizeformat
  from django.utils import timezone
  import pytz
  
  # development/debugging support
  verbose = 2
  def _log(msg):
      if 1 == verbose:
          print(msg)
      elif 2 == verbose:
          f1=open('/tmp/toaster.log', 'a')
          f1.write("|" + msg + "|\n" )
          f1.close()
  
  logger = logging.getLogger("toaster")
  
  
  def error_response(error):
      return JsonResponse({"error": error})
  
  
  class XhrBuildRequest(View):
  
      def get(self, request, *args, **kwargs):
          return HttpResponse()
  
      @staticmethod
      def cancel_build(br):
          """Cancel a build request"""
          try:
              bbctrl = bbcontroller.BitbakeController(br.environment)
              bbctrl.forceShutDown()
          except:
              # We catch a bunch of exceptions here because
              # this is where the server has not had time to start up
              # and the build request or build is in transit between
              # processes.
              # We can safely just set the build as cancelled
              # already as it never got started
              build = br.build
              build.outcome = Build.CANCELLED
              build.save()
  
          # We now hand over to the buildinfohelper to update the
          # build state once we've finished cancelling
          br.state = BuildRequest.REQ_CANCELLING
          br.save()
  
      def post(self, request, *args, **kwargs):
          """
            Build control
  
            Entry point: /xhr_buildrequest/<project_id>
            Method: POST
  
            Args:
                id: id of build to change
                buildCancel = build_request_id ...
                buildDelete = id ...
                targets = recipe_name ...
  
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
  
          project = Project.objects.get(pk=kwargs['pid'])
  
          if 'buildCancel' in request.POST:
              for i in request.POST['buildCancel'].strip().split(" "):
                  try:
                      br = BuildRequest.objects.get(project=project, pk=i)
                      self.cancel_build(br)
                  except BuildRequest.DoesNotExist:
                      return error_response('No such build request id %s' % i)
  
              return error_response('ok')
  
          if 'buildDelete' in request.POST:
              for i in request.POST['buildDelete'].strip().split(" "):
                  try:
                      BuildRequest.objects.select_for_update().get(
                          project=project,
                          pk=i,
                          state__lte=BuildRequest.REQ_DELETED).delete()
  
                  except BuildRequest.DoesNotExist:
                      pass
              return error_response("ok")
  
          if 'targets' in request.POST:
              ProjectTarget.objects.filter(project=project).delete()
              s = str(request.POST['targets'])
              for t in re.sub(r'[;%|"]', '', s).split(" "):
                  if ":" in t:
                      target, task = t.split(":")
                  else:
                      target = t
                      task = ""
                  ProjectTarget.objects.create(project=project,
                                               target=target,
                                               task=task)
              project.schedule_build()
  
              return error_response('ok')
  
          response = HttpResponse()
          response.status_code = 500
          return response
  
  
  class XhrProjectUpdate(View):
  
      def get(self, request, *args, **kwargs):
          return HttpResponse()
  
      def post(self, request, *args, **kwargs):
          """
            Project Update
  
            Entry point: /xhr_projectupdate/<project_id>
            Method: POST
  
            Args:
                pid: pid of project to update
  
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
  
          project = Project.objects.get(pk=kwargs['pid'])
          logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
  
          if 'do_update' in request.POST:
  
              # Extract any default image recipe
              if 'default_image' in request.POST:
                  project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
              else:
                  project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
  
              logger.debug("ProjectUpdateCallback:Chain to the build request")
  
              # Chain to the build request
              xhrBuildRequest = XhrBuildRequest()
              return xhrBuildRequest.post(request, *args, **kwargs)
  
          logger.warning("ERROR:XhrProjectUpdate")
          response = HttpResponse()
          response.status_code = 500
          return response
  
  class XhrSetDefaultImageUrl(View):
  
      def get(self, request, *args, **kwargs):
          return HttpResponse()
  
      def post(self, request, *args, **kwargs):
          """
            Project Update
  
            Entry point: /xhr_setdefaultimage/<project_id>
            Method: POST
  
            Args:
                pid: pid of project to update default image
  
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
  
          project = Project.objects.get(pk=kwargs['pid'])
          logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
  
          # set any default image recipe
          if 'targets' in request.POST:
              default_target = str(request.POST['targets'])
              project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
              logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
              return error_response('ok')
  
          logger.warning("ERROR:XhrSetDefaultImageUrl")
          response = HttpResponse()
          response.status_code = 500
          return response
  
  
  #
  # Layer Management
  #
  # Rules for 'local_source_dir' layers
  #  * Layers must have a unique name in the Layers table
  #  * A 'local_source_dir' layer is supposed to be shared
  #    by all projects that use it, so that it can have the
  #    same logical name
  #  * Each project that uses a layer will have its own
  #    LayerVersion and Project Layer for it
  #  * During the Paroject delete process, when the last
  #    LayerVersion for a 'local_source_dir' layer is deleted
  #    then the Layer record is deleted to remove orphans
  #
  
  def scan_layer_content(layer,layer_version):
      # if this is a local layer directory, we can immediately scan its content
      if layer.local_source_dir:
          try:
              # recipes-*/*/*.bb
              cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
              recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
              recipes_list = recipes_list.decode("utf-8").strip()
              if recipes_list and 'No such' not in recipes_list:
                  for recipe in recipes_list.split('\n'):
                      recipe_path = recipe[recipe.rfind('recipes-'):]
                      recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
                      recipe_ver = recipe_name.rfind('_')
                      if recipe_ver > 0:
                          recipe_name = recipe_name[0:recipe_ver]
                      if recipe_name:
                          ro, created = Recipe.objects.get_or_create(
                              layer_version=layer_version,
                              name=recipe_name
                          )
                          if created:
                              ro.file_path = recipe_path
                              ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
                              ro.description = ro.summary
                          ro.save()
  
          except Exception as e:
              logger.warning("ERROR:scan_layer_content: %s" % e)
  
  class XhrLayer(View):
      """ Delete, Get, Add and Update Layer information
  
          Methods: GET POST DELETE PUT
      """
  
      def get(self, request, *args, **kwargs):
          """
          Get layer information
  
          Method: GET
          Entry point: /xhr_layer/<project id>/<layerversion_id>
          """
  
          try:
              layer_version = Layer_Version.objects.get(
                  pk=kwargs['layerversion_id'])
  
              project = Project.objects.get(pk=kwargs['pid'])
  
              project_layers = ProjectLayer.objects.filter(
                  project=project).values_list("layercommit_id",
                                               flat=True)
  
              ret = {
                  'error': 'ok',
                  'id': layer_version.pk,
                  'name': layer_version.layer.name,
                  'layerdetailurl':
                  layer_version.get_detailspage_url(project.pk),
                  'vcs_ref': layer_version.get_vcs_reference(),
                  'vcs_url': layer_version.layer.vcs_url,
                  'local_source_dir': layer_version.layer.local_source_dir,
                  'layerdeps': {
                      "list": [
                          {
                              "id": dep.id,
                              "name": dep.layer.name,
                              "layerdetailurl":
                              dep.get_detailspage_url(project.pk),
                              "vcs_url": dep.layer.vcs_url,
                              "vcs_reference": dep.get_vcs_reference()
                          }
                          for dep in layer_version.get_alldeps(project.id)]
                  },
                  'projectlayers': list(project_layers)
              }
  
              return JsonResponse(ret)
          except Layer_Version.DoesNotExist:
              error_response("No such layer")
  
      def post(self, request, *args, **kwargs):
          """
            Update a layer
  
            Method: POST
            Entry point: /xhr_layer/<layerversion_id>
  
            Args:
                vcs_url, dirpath, commit, up_branch, summary, description,
                local_source_dir
  
                add_dep = append a layerversion_id as a dependency
                rm_dep = remove a layerversion_id as a depedency
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
  
          try:
              # We currently only allow Imported layers to be edited
              layer_version = Layer_Version.objects.get(
                  id=kwargs['layerversion_id'],
                  project=kwargs['pid'],
                  layer_source=LayerSource.TYPE_IMPORTED)
  
          except Layer_Version.DoesNotExist:
              return error_response("Cannot find imported layer to update")
  
          if "vcs_url" in request.POST:
              layer_version.layer.vcs_url = request.POST["vcs_url"]
          if "dirpath" in request.POST:
              layer_version.dirpath = request.POST["dirpath"]
          if "commit" in request.POST:
              layer_version.commit = request.POST["commit"]
              layer_version.branch = request.POST["commit"]
          if "summary" in request.POST:
              layer_version.layer.summary = request.POST["summary"]
          if "description" in request.POST:
              layer_version.layer.description = request.POST["description"]
          if "local_source_dir" in request.POST:
              layer_version.layer.local_source_dir = \
                  request.POST["local_source_dir"]
  
          if "add_dep" in request.POST:
              lvd = LayerVersionDependency(
                  layer_version=layer_version,
                  depends_on_id=request.POST["add_dep"])
              lvd.save()
  
          if "rm_dep" in request.POST:
              rm_dep = LayerVersionDependency.objects.get(
                  layer_version=layer_version,
                  depends_on_id=request.POST["rm_dep"])
              rm_dep.delete()
  
          try:
              layer_version.layer.save()
              layer_version.save()
          except Exception as e:
              return error_response("Could not update layer version entry: %s"
                                    % e)
  
          return error_response("ok")
  
      def put(self, request, *args, **kwargs):
          """ Add a new layer
  
          Method: PUT
          Entry point: /xhr_layer/<project id>/
          Args:
              project_id, name,
              [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
              (csv)]
  
          """
  
          try:
              project = Project.objects.get(pk=kwargs['pid'])
  
              layer_data = json.loads(request.body.decode('utf-8'))
  
              # We require a unique layer name as otherwise the lists of layers
              # becomes very confusing
              existing_layers = \
                  project.get_all_compatible_layer_versions().values_list(
                      "layer__name",
                      flat=True)
  
              add_to_project = False
              layer_deps_added = []
              if 'add_to_project' in layer_data:
                  add_to_project = True
  
              if layer_data['name'] in existing_layers:
                  return JsonResponse({"error": "layer-name-exists"})
  
              if ('local_source_dir' in layer_data):
                  # Local layer can be shared across projects. They have no 'release'
                  # and are not included in get_all_compatible_layer_versions() above
                  layer,created = Layer.objects.get_or_create(name=layer_data['name'])
                  _log("Local Layer created=%s" % created)
              else:
                  layer = Layer.objects.create(name=layer_data['name'])
  
              layer_version = Layer_Version.objects.create(
                  layer=layer,
                  project=project,
                  layer_source=LayerSource.TYPE_IMPORTED)
  
              # Local layer
              if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
                  layer.local_source_dir = layer_data['local_source_dir']
              # git layer
              elif 'vcs_url' in layer_data:
                  layer.vcs_url = layer_data['vcs_url']
                  layer_version.dirpath = layer_data['dir_path']
                  layer_version.commit = layer_data['git_ref']
                  layer_version.branch = layer_data['git_ref']
  
              layer.save()
              layer_version.save()
  
              if add_to_project:
                  ProjectLayer.objects.get_or_create(
                      layercommit=layer_version, project=project)
  
              # Add the layer dependencies
              if 'layer_deps' in layer_data:
                  for layer_dep_id in layer_data['layer_deps'].split(","):
                      layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
                      LayerVersionDependency.objects.get_or_create(
                          layer_version=layer_version, depends_on=layer_dep)
  
                      # Add layer deps to the project if specified
                      if add_to_project:
                          created, pl = ProjectLayer.objects.get_or_create(
                              layercommit=layer_dep, project=project)
                          layer_deps_added.append(
                              {'name': layer_dep.layer.name,
                               'layerdetailurl':
                               layer_dep.get_detailspage_url(project.pk)})
  
              # Scan the layer's content and update components
              scan_layer_content(layer,layer_version)
  
          except Layer_Version.DoesNotExist:
              return error_response("layer-dep-not-found")
          except Project.DoesNotExist:
              return error_response("project-not-found")
          except KeyError:
              return error_response("incorrect-parameters")
  
          return JsonResponse({'error': "ok",
                               'imported_layer': {
                                   'name': layer.name,
                                   'layerdetailurl':
                                   layer_version.get_detailspage_url()},
                               'deps_added': layer_deps_added})
  
      def delete(self, request, *args, **kwargs):
          """ Delete an imported layer
  
          Method: DELETE
          Entry point: /xhr_layer/<projed id>/<layerversion_id>
  
          """
          try:
              # We currently only allow Imported layers to be deleted
              layer_version = Layer_Version.objects.get(
                  id=kwargs['layerversion_id'],
                  project=kwargs['pid'],
                  layer_source=LayerSource.TYPE_IMPORTED)
          except Layer_Version.DoesNotExist:
              return error_response("Cannot find imported layer to delete")
  
          try:
              ProjectLayer.objects.get(project=kwargs['pid'],
                                       layercommit=layer_version).delete()
          except ProjectLayer.DoesNotExist:
              pass
  
          layer_version.layer.delete()
          layer_version.delete()
  
          return JsonResponse({
              "error": "ok",
              "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],))
          })
  
  
  class XhrCustomRecipe(View):
      """ Create a custom image recipe """
  
      def post(self, request, *args, **kwargs):
          """
          Custom image recipe REST API
  
          Entry point: /xhr_customrecipe/
          Method: POST
  
          Args:
              name: name of custom recipe to create
              project: target project id of orm.models.Project
              base: base recipe id of orm.models.Recipe
  
          Returns:
              {"error": "ok",
               "url": <url of the created recipe>}
              or
              {"error": <error message>}
          """
          # check if request has all required parameters
          for param in ('name', 'project', 'base'):
              if param not in request.POST:
                  return error_response("Missing parameter '%s'" % param)
  
          # get project and baserecipe objects
          params = {}
          for name, model in [("project", Project),
                              ("base", Recipe)]:
              value = request.POST[name]
              try:
                  params[name] = model.objects.get(id=value)
              except model.DoesNotExist:
                  return error_response("Invalid %s id %s" % (name, value))
  
          # create custom recipe
          try:
  
              # Only allowed chars in name are a-z, 0-9 and -
              if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
                  return error_response("invalid-name")
  
              custom_images = CustomImageRecipe.objects.all()
  
              # Are there any recipes with this name already in our project?
              existing_image_recipes_in_project = custom_images.filter(
                  name=request.POST["name"], project=params["project"])
  
              if existing_image_recipes_in_project.count() > 0:
                  return error_response("image-already-exists")
  
              # Are there any recipes with this name which aren't custom
              # image recipes?
              custom_image_ids = custom_images.values_list('id', flat=True)
              existing_non_image_recipes = Recipe.objects.filter(
                  Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
              )
  
              if existing_non_image_recipes.count() > 0:
                  return error_response("recipe-already-exists")
  
              # create layer 'Custom layer' and verion if needed
              layer, l_created = Layer.objects.get_or_create(
                  name=CustomImageRecipe.LAYER_NAME,
                  summary="Layer for custom recipes")
  
              if l_created:
                  layer.local_source_dir = "toaster_created_layer"
                  layer.save()
  
              # Check if we have a layer version already
              # We don't use get_or_create here because the dirpath will change
              # and is a required field
              lver = Layer_Version.objects.filter(Q(project=params['project']) &
                                                  Q(layer=layer) &
                                                  Q(build=None)).last()
              if lver is None:
                  lver, lv_created = Layer_Version.objects.get_or_create(
                      project=params['project'],
                      layer=layer,
                      layer_source=LayerSource.TYPE_LOCAL,
                      dirpath="toaster_created_layer")
  
              # Add a dependency on our layer to the base recipe's layer
              LayerVersionDependency.objects.get_or_create(
                  layer_version=lver,
                  depends_on=params["base"].layer_version)
  
              # Add it to our current project if needed
              ProjectLayer.objects.get_or_create(project=params['project'],
                                                 layercommit=lver,
                                                 optional=False)
  
              # Create the actual recipe
              recipe, r_created = CustomImageRecipe.objects.get_or_create(
                  name=request.POST["name"],
                  base_recipe=params["base"],
                  project=params["project"],
                  layer_version=lver,
                  is_image=True)
  
              # If we created the object then setup these fields. They may get
              # overwritten later on and cause the get_or_create to create a
              # duplicate if they've changed.
              if r_created:
                  recipe.file_path = request.POST["name"]
                  recipe.license = "MIT"
                  recipe.version = "0.1"
                  recipe.save()
  
          except Error as err:
              return error_response("Can't create custom recipe: %s" % err)
  
          # Find the package list from the last build of this recipe/target
          target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
                                         Q(build__project=params['project']) &
                                         (Q(target=params['base'].name) |
                                          Q(target=recipe.name))).last()
          if target:
              # Copy in every package
              # We don't want these packages to be linked to anything because
              # that underlying data may change e.g. delete a build
              for tpackage in target.target_installed_package_set.all():
                  try:
                      built_package = tpackage.package
                      # The package had no recipe information so is a ghost
                      # package skip it
                      if built_package.recipe is None:
                          continue
  
                      config_package = CustomImagePackage.objects.get(
                          name=built_package.name)
  
                      recipe.includes_set.add(config_package)
                  except Exception as e:
                      logger.warning("Error adding package %s %s" %
                                     (tpackage.package.name, e))
                      pass
  
          # pre-create layer directory structure, so that other builds
          # are not blocked by this new recipe dependecy
          # NOTE: this is parallel code to 'localhostbecontroller.py'
          be = BuildEnvironment.objects.all()[0]
          layerpath = os.path.join(be.builddir,
                                   CustomImageRecipe.LAYER_NAME)
          for name in ("conf", "recipes"):
              path = os.path.join(layerpath, name)
              if not os.path.isdir(path):
                  os.makedirs(path)
          # pre-create layer.conf
          config = os.path.join(layerpath, "conf", "layer.conf")
          if not os.path.isfile(config):
              with open(config, "w") as conf:
                  conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
          # pre-create new image's recipe file
          recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
                                     recipe.name)
          with open(recipe_path, "w") as recipef:
              content = recipe.generate_recipe_file_contents()
              if not content:
                  # Delete this incomplete image recipe object
                  recipe.delete()
                  return error_response("recipe-parent-not-exist")
              else:
                  recipef.write(recipe.generate_recipe_file_contents())
  
          return JsonResponse(
              {"error": "ok",
               "packages": recipe.get_all_packages().count(),
               "url": reverse('customrecipe', args=(params['project'].pk,
                                                    recipe.id))})
  
  
  class XhrCustomRecipeId(View):
      """
      Set of ReST API processors working with recipe id.
  
      Entry point: /xhr_customrecipe/<recipe_id>
  
      Methods:
          GET - Get details of custom image recipe
          DELETE - Delete custom image recipe
  
      Returns:
          GET:
          {"error": "ok",
          "info": dictionary of field name -> value pairs
          of the CustomImageRecipe model}
          DELETE:
          {"error": "ok"}
            or
          {"error": <error message>}
      """
      @staticmethod
      def _get_ci_recipe(recipe_id):
          """ Get Custom Image recipe or return an error response"""
          try:
              custom_recipe = \
                  CustomImageRecipe.objects.get(pk=recipe_id)
              return custom_recipe, None
  
          except CustomImageRecipe.DoesNotExist:
              return None, error_response("Custom recipe with id=%s "
                                          "not found" % recipe_id)
  
      def get(self, request, *args, **kwargs):
          custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
          if error:
              return error
  
          if request.method == 'GET':
              info = {"id": custom_recipe.id,
                      "name": custom_recipe.name,
                      "base_recipe_id": custom_recipe.base_recipe.id,
                      "project_id": custom_recipe.project.id}
  
              return JsonResponse({"error": "ok", "info": info})
  
      def delete(self, request, *args, **kwargs):
          custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
          if error:
              return error
  
          project = custom_recipe.project
  
          custom_recipe.delete()
          return JsonResponse({"error": "ok",
                               "gotoUrl": reverse("projectcustomimages",
                                                  args=(project.pk,))})
  
  
  class XhrCustomRecipePackages(View):
      """
      ReST API to add/remove packages to/from custom recipe.
  
      Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
      Methods:
           PUT - Add package to the recipe
           DELETE - Delete package from the recipe
           GET - Get package information
  
       Returns:
           {"error": "ok"}
            or
            {"error": <error message>}
      """
      @staticmethod
      def _get_package(package_id):
          try:
              package = CustomImagePackage.objects.get(pk=package_id)
              return package, None
          except Package.DoesNotExist:
              return None, error_response("Package with id=%s "
                                          "not found" % package_id)
  
      def _traverse_dependents(self, next_package_id,
                               rev_deps, all_current_packages, tree_level=0):
          """
          Recurse through reverse dependency tree for next_package_id.
          Limit the reverse dependency search to packages not already scanned,
          that is, not already in rev_deps.
          Limit the scan to a depth (tree_level) not exceeding the count of
          all packages in the custom image, and if that depth is exceeded
          return False, pop out of the recursion, and write a warning
          to the log, but this is unlikely, suggesting a dependency loop
          not caught by bitbake.
          On return, the input/output arg rev_deps is appended with queryset
          dictionary elements, annotated for use in the customimage template.
          The list has unsorted, but unique elements.
          """
          max_dependency_tree_depth = all_current_packages.count()
          if tree_level >= max_dependency_tree_depth:
              logger.warning(
                  "The number of reverse dependencies "
                  "for this package exceeds " + max_dependency_tree_depth +
                  " and the remaining reverse dependencies will not be removed")
              return True
  
          package = CustomImagePackage.objects.get(id=next_package_id)
          dependents = \
              package.package_dependencies_target.annotate(
                  name=F('package__name'),
                  pk=F('package__pk'),
                  size=F('package__size'),
              ).values("name", "pk", "size").exclude(
                  ~Q(pk__in=all_current_packages)
              )
  
          for pkg in dependents:
              if pkg in rev_deps:
                  # already seen, skip dependent search
                  continue
  
              rev_deps.append(pkg)
              if (self._traverse_dependents(pkg["pk"], rev_deps,
                                            all_current_packages,
                                            tree_level+1)):
                  return True
  
          return False
  
      def _get_all_dependents(self, package_id, all_current_packages):
          """
          Returns sorted list of recursive reverse dependencies for package_id,
          as a list of dictionary items, by recursing through dependency
          relationships.
          """
          rev_deps = []
          self._traverse_dependents(package_id, rev_deps, all_current_packages)
          rev_deps = sorted(rev_deps, key=lambda x: x["name"])
          return rev_deps
  
      def get(self, request, *args, **kwargs):
          recipe, error = XhrCustomRecipeId._get_ci_recipe(
              kwargs['recipe_id'])
          if error:
              return error
  
          # If no package_id then list all the current packages
          if not kwargs['package_id']:
              total_size = 0
              packages = recipe.get_all_packages().values("id",
                                                          "name",
                                                          "version",
                                                          "size")
              for package in packages:
                  package['size_formatted'] = \
                      filtered_filesizeformat(package['size'])
                  total_size += package['size']
  
              return JsonResponse({"error": "ok",
                                   "packages": list(packages),
                                   "total": len(packages),
                                   "total_size": total_size,
                                   "total_size_formatted":
                                   filtered_filesizeformat(total_size)})
          else:
              package, error = XhrCustomRecipePackages._get_package(
                  kwargs['package_id'])
              if error:
                  return error
  
              all_current_packages = recipe.get_all_packages()
  
              # Dependencies for package which aren't satisfied by the
              # current packages in the custom image recipe
              deps = package.package_dependencies_source.for_target_or_none(
                  recipe.name)['packages'].annotate(
                  name=F('depends_on__name'),
                  pk=F('depends_on__pk'),
                  size=F('depends_on__size'),
                  ).values("name", "pk", "size").filter(
                  # There are two depends types we don't know why
                  (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
                   Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
                  ~Q(pk__in=all_current_packages)
                  )
  
              # Reverse dependencies which are needed by packages that are
              # in the image. Recursive search providing all dependents,
              # not just immediate dependents.
              reverse_deps = self._get_all_dependents(kwargs['package_id'],
                                                      all_current_packages)
              total_size_deps = 0
              total_size_reverse_deps = 0
  
              for dep in deps:
                  dep['size_formatted'] = \
                      filtered_filesizeformat(dep['size'])
                  total_size_deps += dep['size']
  
              for dep in reverse_deps:
                  dep['size_formatted'] = \
                      filtered_filesizeformat(dep['size'])
                  total_size_reverse_deps += dep['size']
  
              return JsonResponse(
                  {"error": "ok",
                   "id": package.pk,
                   "name": package.name,
                   "version": package.version,
                   "unsatisfied_dependencies": list(deps),
                   "unsatisfied_dependencies_size": total_size_deps,
                   "unsatisfied_dependencies_size_formatted":
                   filtered_filesizeformat(total_size_deps),
                   "reverse_dependencies": list(reverse_deps),
                   "reverse_dependencies_size": total_size_reverse_deps,
                   "reverse_dependencies_size_formatted":
                   filtered_filesizeformat(total_size_reverse_deps)})
  
      def put(self, request, *args, **kwargs):
          recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
          package, error = self._get_package(kwargs['package_id'])
          if error:
              return error
  
          included_packages = recipe.includes_set.values_list('pk',
                                                              flat=True)
  
          # If we're adding back a package which used to be included in this
          # image all we need to do is remove it from the excludes
          if package.pk in included_packages:
              try:
                  recipe.excludes_set.remove(package)
                  return {"error": "ok"}
              except Package.DoesNotExist:
                  return error_response("Package %s not found in excludes"
                                        " but was in included list" %
                                        package.name)
          else:
              recipe.appends_set.add(package)
              # Make sure that package is not in the excludes set
              try:
                  recipe.excludes_set.remove(package)
              except:
                  pass
  
          # Add the dependencies we think will be added to the recipe
          # as a result of appending this package.
          # TODO this should recurse down the entire deps tree
          for dep in package.package_dependencies_source.all_depends():
              try:
                  cust_package = CustomImagePackage.objects.get(
                      name=dep.depends_on.name)
  
                  recipe.includes_set.add(cust_package)
                  try:
                      # When adding the pre-requisite package, make
                      # sure it's not in the excluded list from a
                      # prior removal.
                      recipe.excludes_set.remove(cust_package)
                  except package.DoesNotExist:
                      # Don't care if the package had never been excluded
                      pass
              except:
                  logger.warning("Could not add package's suggested"
                                 "dependencies to the list")
          return JsonResponse({"error": "ok"})
  
      def delete(self, request, *args, **kwargs):
          recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
          package, error = self._get_package(kwargs['package_id'])
          if error:
              return error
  
          try:
              included_packages = recipe.includes_set.values_list('pk',
                                                                  flat=True)
              # If we're deleting a package which is included we need to
              # Add it to the excludes list.
              if package.pk in included_packages:
                  recipe.excludes_set.add(package)
              else:
                  recipe.appends_set.remove(package)
  
              # remove dependencies as well
              all_current_packages = recipe.get_all_packages()
  
              reverse_deps_dictlist = self._get_all_dependents(
                  package.pk,
                  all_current_packages)
  
              ids = [entry['pk'] for entry in reverse_deps_dictlist]
              reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
              for r in reverse_deps:
                  try:
                      if r.id in included_packages:
                          recipe.excludes_set.add(r)
                      else:
                          recipe.appends_set.remove(r)
                  except:
                      pass
  
              return JsonResponse({"error": "ok"})
          except CustomImageRecipe.DoesNotExist:
              return error_response("Tried to remove package that wasn't"
                                    " present")
  
  
  class XhrProject(View):
      """ Create, delete or edit a project
  
      Entry point: /xhr_project/<project_id>
      """
      def post(self, request, *args, **kwargs):
          """
            Edit project control
  
            Args:
                layerAdd = layer_version_id layer_version_id ...
                layerDel = layer_version_id layer_version_id ...
                projectName = new_project_name
                machineName = new_machine_name
  
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
          try:
              prj = Project.objects.get(pk=kwargs['project_id'])
          except Project.DoesNotExist:
              return error_response("No such project")
  
          # Add layers
          if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0:
              for layer_version_id in request.POST['layerAdd'].split(','):
                  try:
                      lv = Layer_Version.objects.get(pk=int(layer_version_id))
                      ProjectLayer.objects.get_or_create(project=prj,
                                                         layercommit=lv)
                  except Layer_Version.DoesNotExist:
                      return error_response("Layer version %s asked to add "
                                            "doesn't exist" % layer_version_id)
  
          # Remove layers
          if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0:
              layer_version_ids = request.POST['layerDel'].split(',')
              ProjectLayer.objects.filter(
                  project=prj,
                  layercommit_id__in=layer_version_ids).delete()
  
          # Project name change
          if 'projectName' in request.POST:
              prj.name = request.POST['projectName']
              prj.save()
  
          # Machine name change
          if 'machineName' in request.POST:
              machinevar = prj.projectvariable_set.get(name="MACHINE")
              machinevar.value = request.POST['machineName']
              machinevar.save()
  
          # Distro name change
          if 'distroName' in request.POST:
              distrovar = prj.projectvariable_set.get(name="DISTRO")
              distrovar.value = request.POST['distroName']
              distrovar.save()
  
          return JsonResponse({"error": "ok"})
  
      def get(self, request, *args, **kwargs):
          """
          Returns:
              json object representing the current project
          or:
              {"error": <error message>}
          """
  
          try:
              project = Project.objects.get(pk=kwargs['project_id'])
          except Project.DoesNotExist:
              return error_response("Project %s does not exist" %
                                    kwargs['project_id'])
  
          # Create the frequently built targets list
  
          freqtargets = Counter(Target.objects.filter(
              Q(build__project=project),
              ~Q(build__outcome=Build.IN_PROGRESS)
          ).order_by("target").values_list("target", flat=True))
  
          freqtargets = freqtargets.most_common(5)
  
          # We now have the targets in order of frequency but if there are two
          # with the same frequency then we need to make sure those are in
          # alphabetical order without losing the frequency ordering
  
          tmp = []
          switch = None
          for i, freqtartget in enumerate(freqtargets):
              target, count = freqtartget
              try:
                  target_next, count_next = freqtargets[i+1]
                  if count == count_next and target > target_next:
                      switch = target
                      continue
              except IndexError:
                  pass
  
              tmp.append(target)
  
              if switch:
                  tmp.append(switch)
                  switch = None
  
          freqtargets = tmp
  
          layers = []
          for layer in project.projectlayer_set.all():
              layers.append({
                  "id": layer.layercommit.pk,
                  "name": layer.layercommit.layer.name,
                  "vcs_url": layer.layercommit.layer.vcs_url,
                  "local_source_dir": layer.layercommit.layer.local_source_dir,
                  "vcs_reference": layer.layercommit.get_vcs_reference(),
                  "url": layer.layercommit.layer.layer_index_url,
                  "layerdetailurl": layer.layercommit.get_detailspage_url(
                      project.pk),
                  "xhrLayerUrl": reverse("xhr_layer",
                                         args=(project.pk,
                                               layer.layercommit.pk)),
                  "layersource": layer.layercommit.layer_source
              })
  
          data = {
              "name": project.name,
              "layers": layers,
              "freqtargets": freqtargets,
          }
  
          if project.release is not None:
              data['release'] = {
                  "id": project.release.pk,
                  "name": project.release.name,
                  "description": project.release.description
              }
  
          try:
              data["machine"] = {"name":
                                 project.projectvariable_set.get(
                                     name="MACHINE").value}
          except ProjectVariable.DoesNotExist:
              data["machine"] = None
          try:
              data["distro"] = {"name":
                                 project.projectvariable_set.get(
                                     name="DISTRO").value}
          except ProjectVariable.DoesNotExist:
              data["distro"] = None
  
          data['error'] = "ok"
  
          return JsonResponse(data)
  
      def put(self, request, *args, **kwargs):
          # TODO create new project api
          return HttpResponse()
  
      def delete(self, request, *args, **kwargs):
          """Delete a project. Cancels any builds in progress"""
          try:
              project = Project.objects.get(pk=kwargs['project_id'])
              # Cancel any builds in progress
              for br in BuildRequest.objects.filter(
                      project=project,
                      state=BuildRequest.REQ_INPROGRESS):
                  XhrBuildRequest.cancel_build(br)
  
              # gather potential orphaned local layers attached to this project
              project_local_layer_list = []
              for pl in ProjectLayer.objects.filter(project=project):
                  if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
                      project_local_layer_list.append(pl.layercommit.layer)
  
              # deep delete the project and its dependencies
              project.delete()
  
              # delete any local layers now orphaned
              _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
              for layer in project_local_layer_list:
                  layer_refs = Layer_Version.objects.filter(layer=layer)
                  _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
                  if 0 == len(layer_refs):
                      _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
                      Layer.objects.filter(pk=layer.id).delete()
  
          except Project.DoesNotExist:
              return error_response("Project %s does not exist" %
                                    kwargs['project_id'])
  
          return JsonResponse({
              "error": "ok",
              "gotoUrl": reverse("all-projects", args=[])
          })
  
  
  class XhrBuild(View):
      """ Delete a build object
  
      Entry point: /xhr_build/<build_id>
      """
      def delete(self, request, *args, **kwargs):
          """
            Delete build data
  
            Args:
                build_id = build_id
  
            Returns:
                {"error": "ok"}
              or
                {"error": <error message>}
          """
          try:
              build = Build.objects.get(pk=kwargs['build_id'])
              project = build.project
              build.delete()
          except Build.DoesNotExist:
              return error_response("Build %s does not exist" %
                                    kwargs['build_id'])
          return JsonResponse({
              "error": "ok",
              "gotoUrl": reverse("projectbuilds", args=(project.pk,))
          })
  