text: /*
   * SPDX-License-Identifier: MIT
   *
   * Copyright © 2019 Intel Corporation
   */
  
  #include <linux/workqueue.h>
  
  #include "i915_drv.h" /* for_each_engine() */
  #include "i915_request.h"
  #include "intel_engine_heartbeat.h"
  #include "intel_gt.h"
  #include "intel_gt_pm.h"
  #include "intel_gt_requests.h"
  #include "intel_timeline.h"
  
  static bool retire_requests(struct intel_timeline *tl)
  {
  	struct i915_request *rq, *rn;
  
  	list_for_each_entry_safe(rq, rn, &tl->requests, link)
  		if (!i915_request_retire(rq))
  			return false;
  
  	/* And check nothing new was submitted */
  	return !i915_active_fence_isset(&tl->last_request);
  }
  
  static bool engine_active(const struct intel_engine_cs *engine)
  {
  	return !list_empty(&engine->kernel_context->timeline->requests);
  }
  
  static bool flush_submission(struct intel_gt *gt, long timeout)
  {
  	struct intel_engine_cs *engine;
  	enum intel_engine_id id;
  	bool active = false;
  
  	if (!timeout)
  		return false;
  
  	if (!intel_gt_pm_is_awake(gt))
  		return false;
  
  	for_each_engine(engine, gt, id) {
  		intel_engine_flush_submission(engine);
  
  		/* Flush the background retirement and idle barriers */
  		flush_work(&engine->retire_work);
  		flush_delayed_work(&engine->wakeref.work);
  
  		/* Is the idle barrier still outstanding? */
  		active |= engine_active(engine);
  	}
  
  	return active;
  }
  
  static void engine_retire(struct work_struct *work)
  {
  	struct intel_engine_cs *engine =
  		container_of(work, typeof(*engine), retire_work);
  	struct intel_timeline *tl = xchg(&engine->retire, NULL);
  
  	do {
  		struct intel_timeline *next = xchg(&tl->retire, NULL);
  
  		/*
  		 * Our goal here is to retire _idle_ timelines as soon as
  		 * possible (as they are idle, we do not expect userspace
  		 * to be cleaning up anytime soon).
  		 *
  		 * If the timeline is currently locked, either it is being
  		 * retired elsewhere or about to be!
  		 */
  		if (mutex_trylock(&tl->mutex)) {
  			retire_requests(tl);
  			mutex_unlock(&tl->mutex);
  		}
  		intel_timeline_put(tl);
  
  		GEM_BUG_ON(!next);
  		tl = ptr_mask_bits(next, 1);
  	} while (tl);
  }
  
  static bool add_retire(struct intel_engine_cs *engine,
  		       struct intel_timeline *tl)
  {
  #define STUB ((struct intel_timeline *)1)
  	struct intel_timeline *first;
  
  	/*
  	 * We open-code a llist here to include the additional tag [BIT(0)]
  	 * so that we know when the timeline is already on a
  	 * retirement queue: either this engine or another.
  	 */
  
  	if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
  		return false;
  
  	intel_timeline_get(tl);
  	first = READ_ONCE(engine->retire);
  	do
  		tl->retire = ptr_pack_bits(first, 1, 1);
  	while (!try_cmpxchg(&engine->retire, &first, tl));
  
  	return !first;
  }
  
  void intel_engine_add_retire(struct intel_engine_cs *engine,
  			     struct intel_timeline *tl)
  {
  	/* We don't deal well with the engine disappearing beneath us */
  	GEM_BUG_ON(intel_engine_is_virtual(engine));
  
  	if (add_retire(engine, tl))
  		schedule_work(&engine->retire_work);
  }
  
  void intel_engine_init_retire(struct intel_engine_cs *engine)
  {
  	INIT_WORK(&engine->retire_work, engine_retire);
  }
  
  void intel_engine_fini_retire(struct intel_engine_cs *engine)
  {
  	flush_work(&engine->retire_work);
  	GEM_BUG_ON(engine->retire);
  }
  
  long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
  {
  	struct intel_gt_timelines *timelines = &gt->timelines;
  	struct intel_timeline *tl, *tn;
  	unsigned long active_count = 0;
  	bool interruptible;
  	LIST_HEAD(free);
  
  	interruptible = true;
  	if (unlikely(timeout < 0))
  		timeout = -timeout, interruptible = false;
  
  	flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */
  	spin_lock(&timelines->lock);
  	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
  		if (!mutex_trylock(&tl->mutex)) {
  			active_count++; /* report busy to caller, try again? */
  			continue;
  		}
  
  		intel_timeline_get(tl);
  		GEM_BUG_ON(!atomic_read(&tl->active_count));
  		atomic_inc(&tl->active_count); /* pin the list element */
  		spin_unlock(&timelines->lock);
  
  		if (timeout > 0) {
  			struct dma_fence *fence;
  
  			fence = i915_active_fence_get(&tl->last_request);
  			if (fence) {
  				mutex_unlock(&tl->mutex);
  
  				timeout = dma_fence_wait_timeout(fence,
  								 interruptible,
  								 timeout);
  				dma_fence_put(fence);
  
  				/* Retirement is best effort */
  				if (!mutex_trylock(&tl->mutex)) {
  					active_count++;
  					goto out_active;
  				}
  			}
  		}
  
  		if (!retire_requests(tl))
  			active_count++;
  		mutex_unlock(&tl->mutex);
  
  out_active:	spin_lock(&timelines->lock);
  
  		/* Resume list iteration after reacquiring spinlock */
  		list_safe_reset_next(tl, tn, link);
  		if (atomic_dec_and_test(&tl->active_count))
  			list_del(&tl->link);
  
  		/* Defer the final release to after the spinlock */
  		if (refcount_dec_and_test(&tl->kref.refcount)) {
  			GEM_BUG_ON(atomic_read(&tl->active_count));
  			list_add(&tl->link, &free);
  		}
  	}
  	spin_unlock(&timelines->lock);
  
  	list_for_each_entry_safe(tl, tn, &free, link)
  		__intel_timeline_free(&tl->kref);
  
  	if (flush_submission(gt, timeout)) /* Wait, there's more! */
  		active_count++;
  
  	return active_count ? timeout : 0;
  }
  
  int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
  {
  	/* If the device is asleep, we have no requests outstanding */
  	if (!intel_gt_pm_is_awake(gt))
  		return 0;
  
  	while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
  		cond_resched();
  		if (signal_pending(current))
  			return -EINTR;
  	}
  
  	return timeout;
  }
  
  static void retire_work_handler(struct work_struct *work)
  {
  	struct intel_gt *gt =
  		container_of(work, typeof(*gt), requests.retire_work.work);
  
  	schedule_delayed_work(&gt->requests.retire_work,
  			      round_jiffies_up_relative(HZ));
  	intel_gt_retire_requests(gt);
  }
  
  void intel_gt_init_requests(struct intel_gt *gt)
  {
  	INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
  }
  
  void intel_gt_park_requests(struct intel_gt *gt)
  {
  	cancel_delayed_work(&gt->requests.retire_work);
  }
  
  void intel_gt_unpark_requests(struct intel_gt *gt)
  {
  	schedule_delayed_work(&gt->requests.retire_work,
  			      round_jiffies_up_relative(HZ));
  }
  
  void intel_gt_fini_requests(struct intel_gt *gt)
  {
  	/* Wait until the work is marked as finished before unloading! */
  	cancel_delayed_work_sync(&gt->requests.retire_work);
  }
  