2 * SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include <linux/workqueue.h>
9 #include "i915_drv.h" /* for_each_engine() */
10 #include "i915_request.h"
11 #include "intel_engine_heartbeat.h"
13 #include "intel_gt_pm.h"
14 #include "intel_gt_requests.h"
15 #include "intel_timeline.h"
17 static bool retire_requests(struct intel_timeline *tl)
19 struct i915_request *rq, *rn;
21 list_for_each_entry_safe(rq, rn, &tl->requests, link)
22 if (!i915_request_retire(rq))
25 /* And check nothing new was submitted */
26 return !i915_active_fence_isset(&tl->last_request);
29 static bool flush_submission(struct intel_gt *gt)
31 struct intel_engine_cs *engine;
32 enum intel_engine_id id;
35 if (!intel_gt_pm_is_awake(gt))
38 for_each_engine(engine, gt, id) {
39 intel_engine_flush_submission(engine);
40 active |= flush_work(&engine->retire_work);
41 active |= flush_work(&engine->wakeref.work);
47 static void engine_retire(struct work_struct *work)
49 struct intel_engine_cs *engine =
50 container_of(work, typeof(*engine), retire_work);
51 struct intel_timeline *tl = xchg(&engine->retire, NULL);
54 struct intel_timeline *next = xchg(&tl->retire, NULL);
57 * Our goal here is to retire _idle_ timelines as soon as
58 * possible (as they are idle, we do not expect userspace
59 * to be cleaning up anytime soon).
61 * If the timeline is currently locked, either it is being
62 * retired elsewhere or about to be!
64 if (mutex_trylock(&tl->mutex)) {
66 mutex_unlock(&tl->mutex);
68 intel_timeline_put(tl);
71 tl = ptr_mask_bits(next, 1);
75 static bool add_retire(struct intel_engine_cs *engine,
76 struct intel_timeline *tl)
78 #define STUB ((struct intel_timeline *)1)
79 struct intel_timeline *first;
82 * We open-code a llist here to include the additional tag [BIT(0)]
83 * so that we know when the timeline is already on a
84 * retirement queue: either this engine or another.
87 if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
90 intel_timeline_get(tl);
91 first = READ_ONCE(engine->retire);
93 tl->retire = ptr_pack_bits(first, 1, 1);
94 while (!try_cmpxchg(&engine->retire, &first, tl));
99 void intel_engine_add_retire(struct intel_engine_cs *engine,
100 struct intel_timeline *tl)
102 /* We don't deal well with the engine disappearing beneath us */
103 GEM_BUG_ON(intel_engine_is_virtual(engine));
105 if (add_retire(engine, tl))
106 schedule_work(&engine->retire_work);
109 void intel_engine_init_retire(struct intel_engine_cs *engine)
111 INIT_WORK(&engine->retire_work, engine_retire);
114 void intel_engine_fini_retire(struct intel_engine_cs *engine)
116 flush_work(&engine->retire_work);
117 GEM_BUG_ON(engine->retire);
120 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
122 struct intel_gt_timelines *timelines = >->timelines;
123 struct intel_timeline *tl, *tn;
124 unsigned long active_count = 0;
128 interruptible = true;
129 if (unlikely(timeout < 0))
130 timeout = -timeout, interruptible = false;
132 flush_submission(gt); /* kick the ksoftirqd tasklets */
133 spin_lock(&timelines->lock);
134 list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
135 if (!mutex_trylock(&tl->mutex)) {
136 active_count++; /* report busy to caller, try again? */
140 intel_timeline_get(tl);
141 GEM_BUG_ON(!atomic_read(&tl->active_count));
142 atomic_inc(&tl->active_count); /* pin the list element */
143 spin_unlock(&timelines->lock);
146 struct dma_fence *fence;
148 fence = i915_active_fence_get(&tl->last_request);
150 timeout = dma_fence_wait_timeout(fence,
153 dma_fence_put(fence);
157 if (!retire_requests(tl) || flush_submission(gt))
160 spin_lock(&timelines->lock);
162 /* Resume iteration after dropping lock */
163 list_safe_reset_next(tl, tn, link);
164 if (atomic_dec_and_test(&tl->active_count))
167 mutex_unlock(&tl->mutex);
169 /* Defer the final release to after the spinlock */
170 if (refcount_dec_and_test(&tl->kref.refcount)) {
171 GEM_BUG_ON(atomic_read(&tl->active_count));
172 list_add(&tl->link, &free);
175 spin_unlock(&timelines->lock);
177 list_for_each_entry_safe(tl, tn, &free, link)
178 __intel_timeline_free(&tl->kref);
180 return active_count ? timeout : 0;
183 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
185 /* If the device is asleep, we have no requests outstanding */
186 if (!intel_gt_pm_is_awake(gt))
189 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
191 if (signal_pending(current))
198 static void retire_work_handler(struct work_struct *work)
200 struct intel_gt *gt =
201 container_of(work, typeof(*gt), requests.retire_work.work);
203 schedule_delayed_work(>->requests.retire_work,
204 round_jiffies_up_relative(HZ));
205 intel_gt_retire_requests(gt);
208 void intel_gt_init_requests(struct intel_gt *gt)
210 INIT_DELAYED_WORK(>->requests.retire_work, retire_work_handler);
213 void intel_gt_park_requests(struct intel_gt *gt)
215 cancel_delayed_work(>->requests.retire_work);
218 void intel_gt_unpark_requests(struct intel_gt *gt)
220 schedule_delayed_work(>->requests.retire_work,
221 round_jiffies_up_relative(HZ));
224 void intel_gt_fini_requests(struct intel_gt *gt)
226 /* Wait until the work is marked as finished before unloading! */
227 cancel_delayed_work_sync(>->requests.retire_work);