prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  [HOTFIX] Prevent compendia jobs from getting cleaned up and fix QN jobs
  ## Issue Number
  
  #1728 
  #1726 
  #1727 
  
  </issue>
  <code>
  [start of foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py]
  1 import sys
  2 
  3 from django.core.management.base import BaseCommand
  4 
  5 from data_refinery_common.job_lookup import ProcessorPipeline
  6 from data_refinery_common.logging import get_and_configure_logger
  7 from data_refinery_common.message_queue import send_job
  8 from data_refinery_common.models import (Dataset, Experiment, Organism,
  9                                          ProcessorJob,
  10                                          ProcessorJobDatasetAssociation)
  11 from data_refinery_common.utils import queryset_iterator
  12 
  13 logger = get_and_configure_logger(__name__)
  14 
  15 def create_job_for_organism(organism=Organism, quant_sf_only=False, svd_algorithm='ARPACK'):
  16     """Returns a compendia job for the provided organism.
  17 
  18     Fetch all of the experiments and compile large but normally formated Dataset.
  19     """
  20     data = {}
  21     experiments = Experiment.objects.filter(organisms=organism).prefetch_related('samples')
  22 
  23     for experiment in queryset_iterator(experiments):
  24         data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))
  25 
  26     job = ProcessorJob()
  27     job.pipeline_applied = ProcessorPipeline.CREATE_COMPENDIA.value
  28     job.save()
  29 
  30     dset = Dataset()
  31     dset.data = data
  32     dset.scale_by = 'NONE'
  33     dset.aggregate_by = 'SPECIES'
  34     dset.quantile_normalize = False
  35     dset.quant_sf_only = quant_sf_only
  36     dset.svd_algorithm = svd_algorithm
  37     dset.save()
  38 
  39     pjda = ProcessorJobDatasetAssociation()
  40     pjda.processor_job = job
  41     pjda.dataset = dset
  42     pjda.save()
  43 
  44     return job
  45 
  46 
  47 class Command(BaseCommand):
  48 
  49     def add_arguments(self, parser):
  50         parser.add_argument(
  51             "--organisms",
  52             type=str,
  53             help=("Comma separated list of organism names."))
  54 
  55         parser.add_argument(
  56             "--quant-sf-only",
  57             type=lambda x: x == "True",
  58             help=("Whether to create a quantpendium or normal compendium."))
  59 
  60         parser.add_argument(
  61             "--svd-algorithm",
  62             type=str,
  63             help=("Specify SVD algorithm applied during imputation ARPACK, RANDOMIZED or NONE to skip."))
  64 
  65     def handle(self, *args, **options):
  66         """Create a compendium for one or more organisms.
  67 
  68         If --organism is supplied will immediately create a compedium
  69         for it. If not a new job will be dispatched for each organism
  70         with enough microarray samples except for human and mouse.
  71         """
  72         if options["organisms"] is None:
  73             all_organisms = Organism.objects.exclude(name__in=["HOMO_SAPIENS", "MUS_MUSCULUS"])
  74         else:
  75             organisms = options["organisms"].upper().replace(" ", "_").split(",")
  76             all_organisms = Organism.objects.filter(name__in=organisms)
  77 
  78         # I think we could just use options["quant_sf_only"] but I
  79         # wanna make sure that values that are not True do not trigger
  80         # a truthy evaluation.
  81         quant_sf_only = False
  82         if options["quant_sf_only"] is True:
  83             quant_sf_only = True
  84 
  85         # default algorithm to arpack until we decide that ranomized is preferred
  86         svd_algorithm = 'NONE' if quant_sf_only else 'ARPACK'
  87         if options["svd_algorithm"] in ['ARPACK', 'RANDOMIZED', 'NONE']:
  88             svd_algorithm = options["svd_algorithm"]
  89 
  90         logger.debug(all_organisms)
  91 
  92         for organism in all_organisms:
  93             logger.debug(organism)
  94             job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)
  95             logger.info("Sending CREATE_COMPENDIA for Organism", job_id=str(job.pk), organism=str(organism))
  96             send_job(ProcessorPipeline.CREATE_COMPENDIA, job)
  97 
  98         sys.exit(0)
  99
  [end of foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
  --- a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
  +++ b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
  @@ -21,7 +21,8 @@
       experiments = Experiment.objects.filter(organisms=organism).prefetch_related('samples')
   
       for experiment in queryset_iterator(experiments):
  -        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))
  +        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism)\
  +            .values_list('accession_code', flat=True))
   
       job = ProcessorJob()
       job.pipeline_applied = ProcessorPipeline.CREATE_COMPENDIA.value
  @@ -30,7 +31,8 @@
       dset = Dataset()
       dset.data = data
       dset.scale_by = 'NONE'
  -    dset.aggregate_by = 'SPECIES'
  +    # The quantpendias should be aggregated by species
  +    dset.aggregate_by = 'EXPERIMENT' if quant_sf_only else 'SPECIES'
       dset.quantile_normalize = False
       dset.quant_sf_only = quant_sf_only
       dset.svd_algorithm = svd_algorithm
  @@ -55,7 +57,7 @@
           parser.add_argument(
               "--quant-sf-only",
               type=lambda x: x == "True",
  -            help=("Whether to create a quantpendium or normal compendium."))
  +            help=("Whether to create a quantpendium or normal compendium. Quantpendium will be aggregated by EXPERIMENT"))
   
           parser.add_argument(
               "--svd-algorithm",
  @@ -78,19 +80,16 @@
           # I think we could just use options["quant_sf_only"] but I
           # wanna make sure that values that are not True do not trigger
           # a truthy evaluation.
  -        quant_sf_only = False
  -        if options["quant_sf_only"] is True:
  -            quant_sf_only = True
  +        quant_sf_only = options["quant_sf_only"] is True
   
           # default algorithm to arpack until we decide that ranomized is preferred
           svd_algorithm = 'NONE' if quant_sf_only else 'ARPACK'
           if options["svd_algorithm"] in ['ARPACK', 'RANDOMIZED', 'NONE']:
               svd_algorithm = options["svd_algorithm"]
   
  -        logger.debug(all_organisms)
  +        logger.debug('Generating compendia for organisms', organisms=all_organisms)
   
           for organism in all_organisms:
  -            logger.debug(organism)
               job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)
               logger.info("Sending CREATE_COMPENDIA for Organism", job_id=str(job.pk), organism=str(organism))
               send_job(ProcessorPipeline.CREATE_COMPENDIA, job)
  
verification_info: {"golden_diff": "diff --git a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py\n--- a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py\n+++ b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py\n@@ -21,7 +21,8 @@\n     experiments = Experiment.objects.filter(organisms=organism).prefetch_related('samples')\n \n     for experiment in queryset_iterator(experiments):\n-        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))\n+        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism)\\\n+            .values_list('accession_code', flat=True))\n \n     job = ProcessorJob()\n     job.pipeline_applied = ProcessorPipeline.CREATE_COMPENDIA.value\n@@ -30,7 +31,8 @@\n     dset = Dataset()\n     dset.data = data\n     dset.scale_by = 'NONE'\n-    dset.aggregate_by = 'SPECIES'\n+    # The quantpendias should be aggregated by species\n+    dset.aggregate_by = 'EXPERIMENT' if quant_sf_only else 'SPECIES'\n     dset.quantile_normalize = False\n     dset.quant_sf_only = quant_sf_only\n     dset.svd_algorithm = svd_algorithm\n@@ -55,7 +57,7 @@\n         parser.add_argument(\n             \"--quant-sf-only\",\n             type=lambda x: x == \"True\",\n-            help=(\"Whether to create a quantpendium or normal compendium.\"))\n+            help=(\"Whether to create a quantpendium or normal compendium. Quantpendium will be aggregated by EXPERIMENT\"))\n \n         parser.add_argument(\n             \"--svd-algorithm\",\n@@ -78,19 +80,16 @@\n         # I think we could just use options[\"quant_sf_only\"] but I\n         # wanna make sure that values that are not True do not trigger\n         # a truthy evaluation.\n-        quant_sf_only = False\n-        if options[\"quant_sf_only\"] is True:\n-            quant_sf_only = True\n+        quant_sf_only = options[\"quant_sf_only\"] is True\n \n         # default algorithm to arpack until we decide that ranomized is preferred\n         svd_algorithm = 'NONE' if quant_sf_only else 'ARPACK'\n         if options[\"svd_algorithm\"] in ['ARPACK', 'RANDOMIZED', 'NONE']:\n             svd_algorithm = options[\"svd_algorithm\"]\n \n-        logger.debug(all_organisms)\n+        logger.debug('Generating compendia for organisms', organisms=all_organisms)\n \n         for organism in all_organisms:\n-            logger.debug(organism)\n             job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)\n             logger.info(\"Sending CREATE_COMPENDIA for Organism\", job_id=str(job.pk), organism=str(organism))\n             send_job(ProcessorPipeline.CREATE_COMPENDIA, job)\n", "issue": "[HOTFIX] Prevent compendia jobs from getting cleaned up and fix QN jobs\n## Issue Number\r\n\r\n#1728 \r\n#1726 \r\n#1727 \n", "before_files": [{"content": "import sys\n\nfrom django.core.management.base import BaseCommand\n\nfrom data_refinery_common.job_lookup import ProcessorPipeline\nfrom data_refinery_common.logging import get_and_configure_logger\nfrom data_refinery_common.message_queue import send_job\nfrom data_refinery_common.models import (Dataset, Experiment, Organism,\n                                         ProcessorJob,\n                                         ProcessorJobDatasetAssociation)\nfrom data_refinery_common.utils import queryset_iterator\n\nlogger = get_and_configure_logger(__name__)\n\ndef create_job_for_organism(organism=Organism, quant_sf_only=False, svd_algorithm='ARPACK'):\n    \"\"\"Returns a compendia job for the provided organism.\n\n    Fetch all of the experiments and compile large but normally formated Dataset.\n    \"\"\"\n    data = {}\n    experiments = Experiment.objects.filter(organisms=organism).prefetch_related('samples')\n\n    for experiment in queryset_iterator(experiments):\n        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))\n\n    job = ProcessorJob()\n    job.pipeline_applied = ProcessorPipeline.CREATE_COMPENDIA.value\n    job.save()\n\n    dset = Dataset()\n    dset.data = data\n    dset.scale_by = 'NONE'\n    dset.aggregate_by = 'SPECIES'\n    dset.quantile_normalize = False\n    dset.quant_sf_only = quant_sf_only\n    dset.svd_algorithm = svd_algorithm\n    dset.save()\n\n    pjda = ProcessorJobDatasetAssociation()\n    pjda.processor_job = job\n    pjda.dataset = dset\n    pjda.save()\n\n    return job\n\n\nclass Command(BaseCommand):\n\n    def add_arguments(self, parser):\n        parser.add_argument(\n            \"--organisms\",\n            type=str,\n            help=(\"Comma separated list of organism names.\"))\n\n        parser.add_argument(\n            \"--quant-sf-only\",\n            type=lambda x: x == \"True\",\n            help=(\"Whether to create a quantpendium or normal compendium.\"))\n\n        parser.add_argument(\n            \"--svd-algorithm\",\n            type=str,\n            help=(\"Specify SVD algorithm applied during imputation ARPACK, RANDOMIZED or NONE to skip.\"))\n\n    def handle(self, *args, **options):\n        \"\"\"Create a compendium for one or more organisms.\n\n        If --organism is supplied will immediately create a compedium\n        for it. If not a new job will be dispatched for each organism\n        with enough microarray samples except for human and mouse.\n        \"\"\"\n        if options[\"organisms\"] is None:\n            all_organisms = Organism.objects.exclude(name__in=[\"HOMO_SAPIENS\", \"MUS_MUSCULUS\"])\n        else:\n            organisms = options[\"organisms\"].upper().replace(\" \", \"_\").split(\",\")\n            all_organisms = Organism.objects.filter(name__in=organisms)\n\n        # I think we could just use options[\"quant_sf_only\"] but I\n        # wanna make sure that values that are not True do not trigger\n        # a truthy evaluation.\n        quant_sf_only = False\n        if options[\"quant_sf_only\"] is True:\n            quant_sf_only = True\n\n        # default algorithm to arpack until we decide that ranomized is preferred\n        svd_algorithm = 'NONE' if quant_sf_only else 'ARPACK'\n        if options[\"svd_algorithm\"] in ['ARPACK', 'RANDOMIZED', 'NONE']:\n            svd_algorithm = options[\"svd_algorithm\"]\n\n        logger.debug(all_organisms)\n\n        for organism in all_organisms:\n            logger.debug(organism)\n            job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)\n            logger.info(\"Sending CREATE_COMPENDIA for Organism\", job_id=str(job.pk), organism=str(organism))\n            send_job(ProcessorPipeline.CREATE_COMPENDIA, job)\n\n        sys.exit(0)\n", "path": "foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py"}]}