message: Fix kubernetesdeployqueue runs builds out of order
diff: @@ -25,6 +25,15 @@ const {
  initSendToLagoonLogs();
  initSendToLagoonTasks();
  +const pause = duration => new Promise(res => setTimeout(res, duration));
  +
  +const retry = (retries, fn, delay = 1000) =>
  + fn().catch(err =>
  + retries > 1
  + ? pause(delay).then(() => retry(retries - 1, fn, delay))
  + : Promise.reject(err)
  + );
  +
  const messageConsumer = async msg => {
  const {
  buildName,
  @@ -77,29 +86,36 @@ const messageConsumer = async msg => {
  }
  });
  - // Check for currently active builds
  - let activeBuilds;
  - try {
  const jobsGet = promisify(
  kubernetesBatchApi.namespaces(openshiftProject).jobs.get
  );
  - namespaceJobs = await jobsGet();
  - activeBuilds = R.pipe(
  + const hasNoActiveBuilds = () =>
  + new Promise(async (resolve, reject) => {
  + const namespaceJobs = await jobsGet();
  + const activeBuilds = R.pipe(
  R.propOr([], 'items'),
  - R.filter(R.pathSatisfies(R.lt(0), ['status', 'active'])),
  + R.filter(R.pathSatisfies(R.lt(0), ['status', 'active']))
  )(namespaceJobs);
  - } catch (err) {
  - logger.error(
  - `${openshiftProject}: Unexpected error loading jobs, unable to build ${buildName}: ${err}`
  +
  + if (R.isEmpty(activeBuilds)) {
  + resolve();
  + } else {
  + logger.info(
  + `Delaying build of ${buildName} due to ${activeBuilds.length} pending builds`
  );
  - return;
  + reject();
  }
  + });
  - // If there are current builds running, delay this one
  - if (!R.isEmpty(activeBuilds)) {
  - logger.info(`Delaying build of ${buildName} due to ${activeBuilds.length} pending builds`);
  - throw new Error('requeue');
  + // Wait until an there are no active builds in this namespace running
  + try {
  + // Check every minute for 30 minutes
  + await retry(30, hasNoActiveBuilds, 1 * 60 * 1000);
  + } catch (err) {
  + throw new Error(
  + `${openshiftProject}: Requeue build due to error: ${err.message}`
  + );
  }
  // Load job, if not exists create
  