2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/kthread.h>
26 #include <trace/events/dma_fence.h>
27 #include <uapi/linux/sched/types.h>
30 #include "i915_trace.h"
31 #include "intel_gt_pm.h"
32 #include "intel_gt_requests.h"
34 static void irq_enable(struct intel_engine_cs *engine)
36 if (!engine->irq_enable)
39 /* Caller disables interrupts */
40 spin_lock(&engine->gt->irq_lock);
41 engine->irq_enable(engine);
42 spin_unlock(&engine->gt->irq_lock);
45 static void irq_disable(struct intel_engine_cs *engine)
47 if (!engine->irq_disable)
50 /* Caller disables interrupts */
51 spin_lock(&engine->gt->irq_lock);
52 engine->irq_disable(engine);
53 spin_unlock(&engine->gt->irq_lock);
56 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
58 struct intel_engine_cs *engine =
59 container_of(b, struct intel_engine_cs, breadcrumbs);
61 lockdep_assert_held(&b->irq_lock);
63 GEM_BUG_ON(!b->irq_enabled);
64 if (!--b->irq_enabled)
68 intel_gt_pm_put_async(engine->gt);
71 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
73 struct intel_breadcrumbs *b = &engine->breadcrumbs;
79 spin_lock_irqsave(&b->irq_lock, flags);
81 __intel_breadcrumbs_disarm_irq(b);
82 spin_unlock_irqrestore(&b->irq_lock, flags);
85 static inline bool __request_completed(const struct i915_request *rq)
87 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
90 __maybe_unused static bool
91 check_signal_order(struct intel_context *ce, struct i915_request *rq)
93 if (!list_is_last(&rq->signal_link, &ce->signals) &&
94 i915_seqno_passed(rq->fence.seqno,
95 list_next_entry(rq, signal_link)->fence.seqno))
98 if (!list_is_first(&rq->signal_link, &ce->signals) &&
99 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
107 __dma_fence_signal(struct dma_fence *fence)
109 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
113 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
115 fence->timestamp = timestamp;
116 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
117 trace_dma_fence_signaled(fence);
121 __dma_fence_signal__notify(struct dma_fence *fence,
122 const struct list_head *list)
124 struct dma_fence_cb *cur, *tmp;
126 lockdep_assert_held(fence->lock);
128 list_for_each_entry_safe(cur, tmp, list, node) {
129 INIT_LIST_HEAD(&cur->node);
130 cur->func(fence, cur);
134 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
136 struct intel_engine_cs *engine =
137 container_of(b, struct intel_engine_cs, breadcrumbs);
139 if (unlikely(intel_engine_is_virtual(engine)))
140 engine = intel_virtual_engine_get_sibling(engine, 0);
142 intel_engine_add_retire(engine, tl);
145 static void signal_irq_work(struct irq_work *work)
147 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
148 const ktime_t timestamp = ktime_get();
149 struct intel_context *ce, *cn;
150 struct list_head *pos, *next;
153 spin_lock(&b->irq_lock);
155 if (b->irq_armed && list_empty(&b->signalers))
156 __intel_breadcrumbs_disarm_irq(b);
158 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
159 GEM_BUG_ON(list_empty(&ce->signals));
161 list_for_each_safe(pos, next, &ce->signals) {
162 struct i915_request *rq =
163 list_entry(pos, typeof(*rq), signal_link);
165 GEM_BUG_ON(!check_signal_order(ce, rq));
167 if (!__request_completed(rq))
170 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
172 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
174 if (!__dma_fence_signal(&rq->fence))
178 * Queue for execution after dropping the signaling
179 * spinlock as the callback chain may end up adding
180 * more signalers to the same context or engine.
182 i915_request_get(rq);
183 list_add_tail(&rq->signal_link, &signal);
187 * We process the list deletion in bulk, only using a list_add
188 * (not list_move) above but keeping the status of
189 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
191 if (!list_is_first(pos, &ce->signals)) {
192 /* Advance the list to the first incomplete request */
193 __list_del_many(&ce->signals, pos);
194 if (&ce->signals == pos) { /* now empty */
195 list_del_init(&ce->signal_link);
196 add_retire(b, ce->timeline);
201 spin_unlock(&b->irq_lock);
203 list_for_each_safe(pos, next, &signal) {
204 struct i915_request *rq =
205 list_entry(pos, typeof(*rq), signal_link);
206 struct list_head cb_list;
208 spin_lock(&rq->lock);
209 list_replace(&rq->fence.cb_list, &cb_list);
210 __dma_fence_signal__timestamp(&rq->fence, timestamp);
211 __dma_fence_signal__notify(&rq->fence, &cb_list);
212 spin_unlock(&rq->lock);
214 i915_request_put(rq);
218 static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
220 struct intel_engine_cs *engine =
221 container_of(b, struct intel_engine_cs, breadcrumbs);
223 lockdep_assert_held(&b->irq_lock);
227 if (!intel_gt_pm_get_if_awake(engine->gt))
231 * The breadcrumb irq will be disarmed on the interrupt after the
232 * waiters are signaled. This gives us a single interrupt window in
233 * which we can add a new waiter and avoid the cost of re-enabling
239 * Since we are waiting on a request, the GPU should be busy
240 * and should have its own rpm reference. This is tracked
241 * by i915->gt.awake, we can forgo holding our own wakref
242 * for the interrupt as before i915->gt.awake is released (when
243 * the driver is idle) we disarm the breadcrumbs.
246 if (!b->irq_enabled++)
252 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
254 struct intel_breadcrumbs *b = &engine->breadcrumbs;
256 spin_lock_init(&b->irq_lock);
257 INIT_LIST_HEAD(&b->signalers);
259 init_irq_work(&b->irq_work, signal_irq_work);
262 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
264 struct intel_breadcrumbs *b = &engine->breadcrumbs;
267 spin_lock_irqsave(&b->irq_lock, flags);
274 spin_unlock_irqrestore(&b->irq_lock, flags);
277 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
281 bool i915_request_enable_breadcrumb(struct i915_request *rq)
283 lockdep_assert_held(&rq->lock);
285 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
286 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
287 struct intel_context *ce = rq->context;
288 struct list_head *pos;
290 spin_lock(&b->irq_lock);
291 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
293 if (!__intel_breadcrumbs_arm_irq(b))
297 * We keep the seqno in retirement order, so we can break
298 * inside intel_engine_signal_breadcrumbs as soon as we've
299 * passed the last completed request (or seen a request that
300 * hasn't event started). We could walk the timeline->requests,
301 * but keeping a separate signalers_list has the advantage of
302 * hopefully being much smaller than the full list and so
303 * provides faster iteration and detection when there are no
304 * more interrupts required for this context.
306 * We typically expect to add new signalers in order, so we
307 * start looking for our insertion point from the tail of
310 list_for_each_prev(pos, &ce->signals) {
311 struct i915_request *it =
312 list_entry(pos, typeof(*it), signal_link);
314 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
317 list_add(&rq->signal_link, pos);
318 if (pos == &ce->signals) /* catch transitions from empty list */
319 list_move_tail(&ce->signal_link, &b->signalers);
320 GEM_BUG_ON(!check_signal_order(ce, rq));
322 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
324 spin_unlock(&b->irq_lock);
327 return !__request_completed(rq);
330 void i915_request_cancel_breadcrumb(struct i915_request *rq)
332 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
334 lockdep_assert_held(&rq->lock);
337 * We must wait for b->irq_lock so that we know the interrupt handler
338 * has released its reference to the intel_context and has completed
339 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
342 spin_lock(&b->irq_lock);
343 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
344 struct intel_context *ce = rq->context;
346 list_del(&rq->signal_link);
347 if (list_empty(&ce->signals))
348 list_del_init(&ce->signal_link);
350 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
352 spin_unlock(&b->irq_lock);
355 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
356 struct drm_printer *p)
358 struct intel_breadcrumbs *b = &engine->breadcrumbs;
359 struct intel_context *ce;
360 struct i915_request *rq;
362 if (list_empty(&b->signalers))
365 drm_printf(p, "Signals:\n");
367 spin_lock_irq(&b->irq_lock);
368 list_for_each_entry(ce, &b->signalers, signal_link) {
369 list_for_each_entry(rq, &ce->signals, signal_link) {
370 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
371 rq->fence.context, rq->fence.seqno,
372 i915_request_completed(rq) ? "!" :
373 i915_request_started(rq) ? "*" :
375 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
378 spin_unlock_irq(&b->irq_lock);