drivers/gpu/drm/i915/intel_breadcrumbs.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kthread.h>
  26 #include <uapi/linux/sched/types.h>
  27
  28 #include "i915_drv.h"
  29
  30 #ifdef CONFIG_SMP
  31 #define task_asleep(tsk) ((tsk)->state & TASK_NORMAL && !(tsk)->on_cpu)
  32 #else
  33 #define task_asleep(tsk) ((tsk)->state & TASK_NORMAL)
  34 #endif
  35
  36 static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs *b)
  37 {
  38         struct intel_wait *wait;
  39         unsigned int result = 0;
  40
  41         lockdep_assert_held(&b->irq_lock);
  42
  43         wait = b->irq_wait;
  44         if (wait) {
  45                 /*
  46                  * N.B. Since task_asleep() and ttwu are not atomic, the
  47                  * waiter may actually go to sleep after the check, causing
  48                  * us to suppress a valid wakeup. We prefer to reduce the
  49                  * number of false positive missed_breadcrumb() warnings
  50                  * at the expense of a few false negatives, as it it easy
  51                  * to trigger a false positive under heavy load. Enough
  52                  * signal should remain from genuine missed_breadcrumb()
  53                  * for us to detect in CI.
  54                  */
  55                 bool was_asleep = task_asleep(wait->tsk);
  56
  57                 result = ENGINE_WAKEUP_WAITER;
  58                 if (wake_up_process(wait->tsk) && was_asleep)
  59                         result |= ENGINE_WAKEUP_ASLEEP;
  60         }
  61
  62         return result;
  63 }
  64
  65 unsigned int intel_engine_wakeup(struct intel_engine_cs *engine)
  66 {
  67         struct intel_breadcrumbs *b = &engine->breadcrumbs;
  68         unsigned long flags;
  69         unsigned int result;
  70
  71         spin_lock_irqsave(&b->irq_lock, flags);
  72         result = __intel_breadcrumbs_wakeup(b);
  73         spin_unlock_irqrestore(&b->irq_lock, flags);
  74
  75         return result;
  76 }
  77
  78 static unsigned long wait_timeout(void)
  79 {
  80         return round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES);
  81 }
  82
  83 static noinline void missed_breadcrumb(struct intel_engine_cs *engine)
  84 {
  85         if (drm_debug & DRM_UT_DRIVER) {
  86                 struct drm_printer p = drm_debug_printer(__func__);
  87
  88                 intel_engine_dump(engine, &p,
  89                                   "%s missed breadcrumb at %pS\n",
  90                                   engine->name, __builtin_return_address(0));
  91         }
  92
  93         set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
  94 }
  95
  96 static void intel_breadcrumbs_hangcheck(struct timer_list *t)
  97 {
  98         struct intel_engine_cs *engine =
  99                 from_timer(engine, t, breadcrumbs.hangcheck);
 100         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 101
 102         if (!b->irq_armed)
 103                 return;
 104
 105         if (b->hangcheck_interrupts != atomic_read(&engine->irq_count)) {
 106                 b->hangcheck_interrupts = atomic_read(&engine->irq_count);
 107                 mod_timer(&b->hangcheck, wait_timeout());
 108                 return;
 109         }
 110
 111         /* We keep the hangcheck timer alive until we disarm the irq, even
 112          * if there are no waiters at present.
 113          *
 114          * If the waiter was currently running, assume it hasn't had a chance
 115          * to process the pending interrupt (e.g, low priority task on a loaded
 116          * system) and wait until it sleeps before declaring a missed interrupt.
 117          *
 118          * If the waiter was asleep (and not even pending a wakeup), then we
 119          * must have missed an interrupt as the GPU has stopped advancing
 120          * but we still have a waiter. Assuming all batches complete within
 121          * DRM_I915_HANGCHECK_JIFFIES [1.5s]!
 122          */
 123         if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP) {
 124                 missed_breadcrumb(engine);
 125                 mod_timer(&b->fake_irq, jiffies + 1);
 126         } else {
 127                 mod_timer(&b->hangcheck, wait_timeout());
 128         }
 129 }
 130
 131 static void intel_breadcrumbs_fake_irq(struct timer_list *t)
 132 {
 133         struct intel_engine_cs *engine = from_timer(engine, t,
 134                                                     breadcrumbs.fake_irq);
 135         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 136
 137         /* The timer persists in case we cannot enable interrupts,
 138          * or if we have previously seen seqno/interrupt incoherency
 139          * ("missed interrupt" syndrome, better known as a "missed breadcrumb").
 140          * Here the worker will wake up every jiffie in order to kick the
 141          * oldest waiter to do the coherent seqno check.
 142          */
 143
 144         spin_lock_irq(&b->irq_lock);
 145         if (b->irq_armed && !__intel_breadcrumbs_wakeup(b))
 146                 __intel_engine_disarm_breadcrumbs(engine);
 147         spin_unlock_irq(&b->irq_lock);
 148         if (!b->irq_armed)
 149                 return;
 150
 151         mod_timer(&b->fake_irq, jiffies + 1);
 152 }
 153
 154 static void irq_enable(struct intel_engine_cs *engine)
 155 {
 156         /*
 157          * FIXME: Ideally we want this on the API boundary, but for the
 158          * sake of testing with mock breadcrumbs (no HW so unable to
 159          * enable irqs) we place it deep within the bowels, at the point
 160          * of no return.
 161          */
 162         GEM_BUG_ON(!intel_irqs_enabled(engine->i915));
 163
 164         /* Enabling the IRQ may miss the generation of the interrupt, but
 165          * we still need to force the barrier before reading the seqno,
 166          * just in case.
 167          */
 168         set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 169
 170         /* Caller disables interrupts */
 171         spin_lock(&engine->i915->irq_lock);
 172         engine->irq_enable(engine);
 173         spin_unlock(&engine->i915->irq_lock);
 174 }
 175
 176 static void irq_disable(struct intel_engine_cs *engine)
 177 {
 178         /* Caller disables interrupts */
 179         spin_lock(&engine->i915->irq_lock);
 180         engine->irq_disable(engine);
 181         spin_unlock(&engine->i915->irq_lock);
 182 }
 183
 184 void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 185 {
 186         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 187
 188         lockdep_assert_held(&b->irq_lock);
 189         GEM_BUG_ON(b->irq_wait);
 190         GEM_BUG_ON(!b->irq_armed);
 191
 192         GEM_BUG_ON(!b->irq_enabled);
 193         if (!--b->irq_enabled)
 194                 irq_disable(engine);
 195
 196         b->irq_armed = false;
 197 }
 198
 199 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
 200 {
 201         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 202
 203         spin_lock_irq(&b->irq_lock);
 204         if (!b->irq_enabled++)
 205                 irq_enable(engine);
 206         GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
 207         spin_unlock_irq(&b->irq_lock);
 208 }
 209
 210 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
 211 {
 212         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 213
 214         spin_lock_irq(&b->irq_lock);
 215         GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
 216         if (!--b->irq_enabled)
 217                 irq_disable(engine);
 218         spin_unlock_irq(&b->irq_lock);
 219 }
 220
 221 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 222 {
 223         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 224         struct intel_wait *wait, *n;
 225
 226         if (!b->irq_armed)
 227                 goto wakeup_signaler;
 228
 229         /*
 230          * We only disarm the irq when we are idle (all requests completed),
 231          * so if the bottom-half remains asleep, it missed the request
 232          * completion.
 233          */
 234         if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP)
 235                 missed_breadcrumb(engine);
 236
 237         spin_lock_irq(&b->rb_lock);
 238
 239         spin_lock(&b->irq_lock);
 240         b->irq_wait = NULL;
 241         if (b->irq_armed)
 242                 __intel_engine_disarm_breadcrumbs(engine);
 243         spin_unlock(&b->irq_lock);
 244
 245         rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {
 246                 RB_CLEAR_NODE(&wait->node);
 247                 wake_up_process(wait->tsk);
 248         }
 249         b->waiters = RB_ROOT;
 250
 251         spin_unlock_irq(&b->rb_lock);
 252
 253         /*
 254          * The signaling thread may be asleep holding a reference to a request,
 255          * that had its signaling cancelled prior to being preempted. We need
 256          * to kick the signaler, just in case, to release any such reference.
 257          */
 258 wakeup_signaler:
 259         wake_up_process(b->signaler);
 260 }
 261
 262 static bool use_fake_irq(const struct intel_breadcrumbs *b)
 263 {
 264         const struct intel_engine_cs *engine =
 265                 container_of(b, struct intel_engine_cs, breadcrumbs);
 266
 267         if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings))
 268                 return false;
 269
 270         /* Only start with the heavy weight fake irq timer if we have not
 271          * seen any interrupts since enabling it the first time. If the
 272          * interrupts are still arriving, it means we made a mistake in our
 273          * engine->seqno_barrier(), a timing error that should be transient
 274          * and unlikely to reoccur.
 275          */
 276         return atomic_read(&engine->irq_count) == b->hangcheck_interrupts;
 277 }
 278
 279 static void enable_fake_irq(struct intel_breadcrumbs *b)
 280 {
 281         /* Ensure we never sleep indefinitely */
 282         if (!b->irq_enabled || use_fake_irq(b))
 283                 mod_timer(&b->fake_irq, jiffies + 1);
 284         else
 285                 mod_timer(&b->hangcheck, wait_timeout());
 286 }
 287
 288 static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 289 {
 290         struct intel_engine_cs *engine =
 291                 container_of(b, struct intel_engine_cs, breadcrumbs);
 292         struct drm_i915_private *i915 = engine->i915;
 293         bool enabled;
 294
 295         lockdep_assert_held(&b->irq_lock);
 296         if (b->irq_armed)
 297                 return false;
 298
 299         /* The breadcrumb irq will be disarmed on the interrupt after the
 300          * waiters are signaled. This gives us a single interrupt window in
 301          * which we can add a new waiter and avoid the cost of re-enabling
 302          * the irq.
 303          */
 304         b->irq_armed = true;
 305
 306         if (I915_SELFTEST_ONLY(b->mock)) {
 307                 /* For our mock objects we want to avoid interaction
 308                  * with the real hardware (which is not set up). So
 309                  * we simply pretend we have enabled the powerwell
 310                  * and the irq, and leave it up to the mock
 311                  * implementation to call intel_engine_wakeup()
 312                  * itself when it wants to simulate a user interrupt,
 313                  */
 314                 return true;
 315         }
 316
 317         /* Since we are waiting on a request, the GPU should be busy
 318          * and should have its own rpm reference. This is tracked
 319          * by i915->gt.awake, we can forgo holding our own wakref
 320          * for the interrupt as before i915->gt.awake is released (when
 321          * the driver is idle) we disarm the breadcrumbs.
 322          */
 323
 324         /* No interrupts? Kick the waiter every jiffie! */
 325         enabled = false;
 326         if (!b->irq_enabled++ &&
 327             !test_bit(engine->id, &i915->gpu_error.test_irq_rings)) {
 328                 irq_enable(engine);
 329                 enabled = true;
 330         }
 331
 332         enable_fake_irq(b);
 333         return enabled;
 334 }
 335
 336 static inline struct intel_wait *to_wait(struct rb_node *node)
 337 {
 338         return rb_entry(node, struct intel_wait, node);
 339 }
 340
 341 static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,
 342                                               struct intel_wait *wait)
 343 {
 344         lockdep_assert_held(&b->rb_lock);
 345         GEM_BUG_ON(b->irq_wait == wait);
 346
 347         /* This request is completed, so remove it from the tree, mark it as
 348          * complete, and *then* wake up the associated task. N.B. when the
 349          * task wakes up, it will find the empty rb_node, discern that it
 350          * has already been removed from the tree and skip the serialisation
 351          * of the b->rb_lock and b->irq_lock. This means that the destruction
 352          * of the intel_wait is not serialised with the interrupt handler
 353          * by the waiter - it must instead be serialised by the caller.
 354          */
 355         rb_erase(&wait->node, &b->waiters);
 356         RB_CLEAR_NODE(&wait->node);
 357
 358         wake_up_process(wait->tsk); /* implicit smp_wmb() */
 359 }
 360
 361 static inline void __intel_breadcrumbs_next(struct intel_engine_cs *engine,
 362                                             struct rb_node *next)
 363 {
 364         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 365
 366         spin_lock(&b->irq_lock);
 367         GEM_BUG_ON(!b->irq_armed);
 368         GEM_BUG_ON(!b->irq_wait);
 369         b->irq_wait = to_wait(next);
 370         spin_unlock(&b->irq_lock);
 371
 372         /* We always wake up the next waiter that takes over as the bottom-half
 373          * as we may delegate not only the irq-seqno barrier to the next waiter
 374          * but also the task of waking up concurrent waiters.
 375          */
 376         if (next)
 377                 wake_up_process(to_wait(next)->tsk);
 378 }
 379
 380 static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 381                                     struct intel_wait *wait)
 382 {
 383         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 384         struct rb_node **p, *parent, *completed;
 385         bool first, armed;
 386         u32 seqno;
 387
 388         /* Insert the request into the retirement ordered list
 389          * of waiters by walking the rbtree. If we are the oldest
 390          * seqno in the tree (the first to be retired), then
 391          * set ourselves as the bottom-half.
 392          *
 393          * As we descend the tree, prune completed branches since we hold the
 394          * spinlock we know that the first_waiter must be delayed and can
 395          * reduce some of the sequential wake up latency if we take action
 396          * ourselves and wake up the completed tasks in parallel. Also, by
 397          * removing stale elements in the tree, we may be able to reduce the
 398          * ping-pong between the old bottom-half and ourselves as first-waiter.
 399          */
 400         armed = false;
 401         first = true;
 402         parent = NULL;
 403         completed = NULL;
 404         seqno = intel_engine_get_seqno(engine);
 405
 406          /* If the request completed before we managed to grab the spinlock,
 407           * return now before adding ourselves to the rbtree. We let the
 408           * current bottom-half handle any pending wakeups and instead
 409           * try and get out of the way quickly.
 410           */
 411         if (i915_seqno_passed(seqno, wait->seqno)) {
 412                 RB_CLEAR_NODE(&wait->node);
 413                 return first;
 414         }
 415
 416         p = &b->waiters.rb_node;
 417         while (*p) {
 418                 parent = *p;
 419                 if (wait->seqno == to_wait(parent)->seqno) {
 420                         /* We have multiple waiters on the same seqno, select
 421                          * the highest priority task (that with the smallest
 422                          * task->prio) to serve as the bottom-half for this
 423                          * group.
 424                          */
 425                         if (wait->tsk->prio > to_wait(parent)->tsk->prio) {
 426                                 p = &parent->rb_right;
 427                                 first = false;
 428                         } else {
 429                                 p = &parent->rb_left;
 430                         }
 431                 } else if (i915_seqno_passed(wait->seqno,
 432                                              to_wait(parent)->seqno)) {
 433                         p = &parent->rb_right;
 434                         if (i915_seqno_passed(seqno, to_wait(parent)->seqno))
 435                                 completed = parent;
 436                         else
 437                                 first = false;
 438                 } else {
 439                         p = &parent->rb_left;
 440                 }
 441         }
 442         rb_link_node(&wait->node, parent, p);
 443         rb_insert_color(&wait->node, &b->waiters);
 444
 445         if (first) {
 446                 spin_lock(&b->irq_lock);
 447                 b->irq_wait = wait;
 448                 /* After assigning ourselves as the new bottom-half, we must
 449                  * perform a cursory check to prevent a missed interrupt.
 450                  * Either we miss the interrupt whilst programming the hardware,
 451                  * or if there was a previous waiter (for a later seqno) they
 452                  * may be woken instead of us (due to the inherent race
 453                  * in the unlocked read of b->irq_seqno_bh in the irq handler)
 454                  * and so we miss the wake up.
 455                  */
 456                 armed = __intel_breadcrumbs_enable_irq(b);
 457                 spin_unlock(&b->irq_lock);
 458         }
 459
 460         if (completed) {
 461                 /* Advance the bottom-half (b->irq_wait) before we wake up
 462                  * the waiters who may scribble over their intel_wait
 463                  * just as the interrupt handler is dereferencing it via
 464                  * b->irq_wait.
 465                  */
 466                 if (!first) {
 467                         struct rb_node *next = rb_next(completed);
 468                         GEM_BUG_ON(next == &wait->node);
 469                         __intel_breadcrumbs_next(engine, next);
 470                 }
 471
 472                 do {
 473                         struct intel_wait *crumb = to_wait(completed);
 474                         completed = rb_prev(completed);
 475                         __intel_breadcrumbs_finish(b, crumb);
 476                 } while (completed);
 477         }
 478
 479         GEM_BUG_ON(!b->irq_wait);
 480         GEM_BUG_ON(!b->irq_armed);
 481         GEM_BUG_ON(rb_first(&b->waiters) != &b->irq_wait->node);
 482
 483         return armed;
 484 }
 485
 486 bool intel_engine_add_wait(struct intel_engine_cs *engine,
 487                            struct intel_wait *wait)
 488 {
 489         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 490         bool armed;
 491
 492         spin_lock_irq(&b->rb_lock);
 493         armed = __intel_engine_add_wait(engine, wait);
 494         spin_unlock_irq(&b->rb_lock);
 495         if (armed)
 496                 return armed;
 497
 498         /* Make the caller recheck if its request has already started. */
 499         return i915_seqno_passed(intel_engine_get_seqno(engine),
 500                                  wait->seqno - 1);
 501 }
 502
 503 static inline bool chain_wakeup(struct rb_node *rb, int priority)
 504 {
 505         return rb && to_wait(rb)->tsk->prio <= priority;
 506 }
 507
 508 static inline int wakeup_priority(struct intel_breadcrumbs *b,
 509                                   struct task_struct *tsk)
 510 {
 511         if (tsk == b->signaler)
 512                 return INT_MIN;
 513         else
 514                 return tsk->prio;
 515 }
 516
 517 static void __intel_engine_remove_wait(struct intel_engine_cs *engine,
 518                                        struct intel_wait *wait)
 519 {
 520         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 521
 522         lockdep_assert_held(&b->rb_lock);
 523
 524         if (RB_EMPTY_NODE(&wait->node))
 525                 goto out;
 526
 527         if (b->irq_wait == wait) {
 528                 const int priority = wakeup_priority(b, wait->tsk);
 529                 struct rb_node *next;
 530
 531                 /* We are the current bottom-half. Find the next candidate,
 532                  * the first waiter in the queue on the remaining oldest
 533                  * request. As multiple seqnos may complete in the time it
 534                  * takes us to wake up and find the next waiter, we have to
 535                  * wake up that waiter for it to perform its own coherent
 536                  * completion check.
 537                  */
 538                 next = rb_next(&wait->node);
 539                 if (chain_wakeup(next, priority)) {
 540                         /* If the next waiter is already complete,
 541                          * wake it up and continue onto the next waiter. So
 542                          * if have a small herd, they will wake up in parallel
 543                          * rather than sequentially, which should reduce
 544                          * the overall latency in waking all the completed
 545                          * clients.
 546                          *
 547                          * However, waking up a chain adds extra latency to
 548                          * the first_waiter. This is undesirable if that
 549                          * waiter is a high priority task.
 550                          */
 551                         u32 seqno = intel_engine_get_seqno(engine);
 552
 553                         while (i915_seqno_passed(seqno, to_wait(next)->seqno)) {
 554                                 struct rb_node *n = rb_next(next);
 555
 556                                 __intel_breadcrumbs_finish(b, to_wait(next));
 557                                 next = n;
 558                                 if (!chain_wakeup(next, priority))
 559                                         break;
 560                         }
 561                 }
 562
 563                 __intel_breadcrumbs_next(engine, next);
 564         } else {
 565                 GEM_BUG_ON(rb_first(&b->waiters) == &wait->node);
 566         }
 567
 568         GEM_BUG_ON(RB_EMPTY_NODE(&wait->node));
 569         rb_erase(&wait->node, &b->waiters);
 570         RB_CLEAR_NODE(&wait->node);
 571
 572 out:
 573         GEM_BUG_ON(b->irq_wait == wait);
 574         GEM_BUG_ON(rb_first(&b->waiters) !=
 575                    (b->irq_wait ? &b->irq_wait->node : NULL));
 576 }
 577
 578 void intel_engine_remove_wait(struct intel_engine_cs *engine,
 579                               struct intel_wait *wait)
 580 {
 581         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 582
 583         /* Quick check to see if this waiter was already decoupled from
 584          * the tree by the bottom-half to avoid contention on the spinlock
 585          * by the herd.
 586          */
 587         if (RB_EMPTY_NODE(&wait->node)) {
 588                 GEM_BUG_ON(READ_ONCE(b->irq_wait) == wait);
 589                 return;
 590         }
 591
 592         spin_lock_irq(&b->rb_lock);
 593         __intel_engine_remove_wait(engine, wait);
 594         spin_unlock_irq(&b->rb_lock);
 595 }
 596
 597 static bool signal_complete(const struct drm_i915_gem_request *request)
 598 {
 599         if (!request)
 600                 return false;
 601
 602         /*
 603          * Carefully check if the request is complete, giving time for the
 604          * seqno to be visible or if the GPU hung.
 605          */
 606         return __i915_request_irq_complete(request);
 607 }
 608
 609 static struct drm_i915_gem_request *to_signaler(struct rb_node *rb)
 610 {
 611         return rb_entry(rb, struct drm_i915_gem_request, signaling.node);
 612 }
 613
 614 static void signaler_set_rtpriority(void)
 615 {
 616          struct sched_param param = { .sched_priority = 1 };
 617
 618          sched_setscheduler_nocheck(current, SCHED_FIFO, &param);
 619 }
 620
 621 static int intel_breadcrumbs_signaler(void *arg)
 622 {
 623         struct intel_engine_cs *engine = arg;
 624         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 625         struct drm_i915_gem_request *request;
 626
 627         /* Install ourselves with high priority to reduce signalling latency */
 628         signaler_set_rtpriority();
 629
 630         do {
 631                 bool do_schedule = true;
 632
 633                 set_current_state(TASK_INTERRUPTIBLE);
 634
 635                 /* We are either woken up by the interrupt bottom-half,
 636                  * or by a client adding a new signaller. In both cases,
 637                  * the GPU seqno may have advanced beyond our oldest signal.
 638                  * If it has, propagate the signal, remove the waiter and
 639                  * check again with the next oldest signal. Otherwise we
 640                  * need to wait for a new interrupt from the GPU or for
 641                  * a new client.
 642                  */
 643                 rcu_read_lock();
 644                 request = rcu_dereference(b->first_signal);
 645                 if (request)
 646                         request = i915_gem_request_get_rcu(request);
 647                 rcu_read_unlock();
 648                 if (signal_complete(request)) {
 649                         if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 650                                       &request->fence.flags)) {
 651                                 local_bh_disable();
 652                                 dma_fence_signal(&request->fence);
 653                                 GEM_BUG_ON(!i915_gem_request_completed(request));
 654                                 local_bh_enable(); /* kick start the tasklets */
 655                         }
 656
 657                         spin_lock_irq(&b->rb_lock);
 658
 659                         /* Wake up all other completed waiters and select the
 660                          * next bottom-half for the next user interrupt.
 661                          */
 662                         __intel_engine_remove_wait(engine,
 663                                                    &request->signaling.wait);
 664
 665                         /* Find the next oldest signal. Note that as we have
 666                          * not been holding the lock, another client may
 667                          * have installed an even older signal than the one
 668                          * we just completed - so double check we are still
 669                          * the oldest before picking the next one.
 670                          */
 671                         if (request == rcu_access_pointer(b->first_signal)) {
 672                                 struct rb_node *rb =
 673                                         rb_next(&request->signaling.node);
 674                                 rcu_assign_pointer(b->first_signal,
 675                                                    rb ? to_signaler(rb) : NULL);
 676                         }
 677                         rb_erase(&request->signaling.node, &b->signals);
 678                         RB_CLEAR_NODE(&request->signaling.node);
 679
 680                         spin_unlock_irq(&b->rb_lock);
 681
 682                         i915_gem_request_put(request);
 683
 684                         /* If the engine is saturated we may be continually
 685                          * processing completed requests. This angers the
 686                          * NMI watchdog if we never let anything else
 687                          * have access to the CPU. Let's pretend to be nice
 688                          * and relinquish the CPU if we burn through the
 689                          * entire RT timeslice!
 690                          */
 691                         do_schedule = need_resched();
 692                 }
 693
 694                 if (unlikely(do_schedule)) {
 695                         if (kthread_should_park())
 696                                 kthread_parkme();
 697
 698                         if (unlikely(kthread_should_stop())) {
 699                                 i915_gem_request_put(request);
 700                                 break;
 701                         }
 702
 703                         schedule();
 704                 }
 705                 i915_gem_request_put(request);
 706         } while (1);
 707         __set_current_state(TASK_RUNNING);
 708
 709         return 0;
 710 }
 711
 712 void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
 713                                    bool wakeup)
 714 {
 715         struct intel_engine_cs *engine = request->engine;
 716         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 717         u32 seqno;
 718
 719         /* Note that we may be called from an interrupt handler on another
 720          * device (e.g. nouveau signaling a fence completion causing us
 721          * to submit a request, and so enable signaling). As such,
 722          * we need to make sure that all other users of b->rb_lock protect
 723          * against interrupts, i.e. use spin_lock_irqsave.
 724          */
 725
 726         /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */
 727         GEM_BUG_ON(!irqs_disabled());
 728         lockdep_assert_held(&request->lock);
 729
 730         seqno = i915_gem_request_global_seqno(request);
 731         if (!seqno)
 732                 return;
 733
 734         request->signaling.wait.tsk = b->signaler;
 735         request->signaling.wait.request = request;
 736         request->signaling.wait.seqno = seqno;
 737         i915_gem_request_get(request);
 738
 739         spin_lock(&b->rb_lock);
 740
 741         /* First add ourselves into the list of waiters, but register our
 742          * bottom-half as the signaller thread. As per usual, only the oldest
 743          * waiter (not just signaller) is tasked as the bottom-half waking
 744          * up all completed waiters after the user interrupt.
 745          *
 746          * If we are the oldest waiter, enable the irq (after which we
 747          * must double check that the seqno did not complete).
 748          */
 749         wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait);
 750
 751         if (!__i915_gem_request_completed(request, seqno)) {
 752                 struct rb_node *parent, **p;
 753                 bool first;
 754
 755                 /* Now insert ourselves into the retirement ordered list of
 756                  * signals on this engine. We track the oldest seqno as that
 757                  * will be the first signal to complete.
 758                  */
 759                 parent = NULL;
 760                 first = true;
 761                 p = &b->signals.rb_node;
 762                 while (*p) {
 763                         parent = *p;
 764                         if (i915_seqno_passed(seqno,
 765                                               to_signaler(parent)->signaling.wait.seqno)) {
 766                                 p = &parent->rb_right;
 767                                 first = false;
 768                         } else {
 769                                 p = &parent->rb_left;
 770                         }
 771                 }
 772                 rb_link_node(&request->signaling.node, parent, p);
 773                 rb_insert_color(&request->signaling.node, &b->signals);
 774                 if (first)
 775                         rcu_assign_pointer(b->first_signal, request);
 776         } else {
 777                 __intel_engine_remove_wait(engine, &request->signaling.wait);
 778                 i915_gem_request_put(request);
 779                 wakeup = false;
 780         }
 781
 782         spin_unlock(&b->rb_lock);
 783
 784         if (wakeup)
 785                 wake_up_process(b->signaler);
 786 }
 787
 788 void intel_engine_cancel_signaling(struct drm_i915_gem_request *request)
 789 {
 790         struct intel_engine_cs *engine = request->engine;
 791         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 792
 793         GEM_BUG_ON(!irqs_disabled());
 794         lockdep_assert_held(&request->lock);
 795         GEM_BUG_ON(!request->signaling.wait.seqno);
 796
 797         spin_lock(&b->rb_lock);
 798
 799         if (!RB_EMPTY_NODE(&request->signaling.node)) {
 800                 if (request == rcu_access_pointer(b->first_signal)) {
 801                         struct rb_node *rb =
 802                                 rb_next(&request->signaling.node);
 803                         rcu_assign_pointer(b->first_signal,
 804                                            rb ? to_signaler(rb) : NULL);
 805                 }
 806                 rb_erase(&request->signaling.node, &b->signals);
 807                 RB_CLEAR_NODE(&request->signaling.node);
 808                 i915_gem_request_put(request);
 809         }
 810
 811         __intel_engine_remove_wait(engine, &request->signaling.wait);
 812
 813         spin_unlock(&b->rb_lock);
 814
 815         request->signaling.wait.seqno = 0;
 816 }
 817
 818 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 819 {
 820         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 821         struct task_struct *tsk;
 822
 823         spin_lock_init(&b->rb_lock);
 824         spin_lock_init(&b->irq_lock);
 825
 826         timer_setup(&b->fake_irq, intel_breadcrumbs_fake_irq, 0);
 827         timer_setup(&b->hangcheck, intel_breadcrumbs_hangcheck, 0);
 828
 829         /* Spawn a thread to provide a common bottom-half for all signals.
 830          * As this is an asynchronous interface we cannot steal the current
 831          * task for handling the bottom-half to the user interrupt, therefore
 832          * we create a thread to do the coherent seqno dance after the
 833          * interrupt and then signal the waitqueue (via the dma-buf/fence).
 834          */
 835         tsk = kthread_run(intel_breadcrumbs_signaler, engine,
 836                           "i915/signal:%d", engine->id);
 837         if (IS_ERR(tsk))
 838                 return PTR_ERR(tsk);
 839
 840         b->signaler = tsk;
 841
 842         return 0;
 843 }
 844
 845 static void cancel_fake_irq(struct intel_engine_cs *engine)
 846 {
 847         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 848
 849         del_timer_sync(&b->hangcheck);
 850         del_timer_sync(&b->fake_irq);
 851         clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
 852 }
 853
 854 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 855 {
 856         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 857
 858         cancel_fake_irq(engine);
 859         spin_lock_irq(&b->irq_lock);
 860
 861         if (b->irq_enabled)
 862                 irq_enable(engine);
 863         else
 864                 irq_disable(engine);
 865
 866         /* We set the IRQ_BREADCRUMB bit when we enable the irq presuming the
 867          * GPU is active and may have already executed the MI_USER_INTERRUPT
 868          * before the CPU is ready to receive. However, the engine is currently
 869          * idle (we haven't started it yet), there is no possibility for a
 870          * missed interrupt as we enabled the irq and so we can clear the
 871          * immediate wakeup (until a real interrupt arrives for the waiter).
 872          */
 873         clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 874
 875         if (b->irq_armed)
 876                 enable_fake_irq(b);
 877
 878         spin_unlock_irq(&b->irq_lock);
 879 }
 880
 881 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 882 {
 883         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 884
 885         /* The engines should be idle and all requests accounted for! */
 886         WARN_ON(READ_ONCE(b->irq_wait));
 887         WARN_ON(!RB_EMPTY_ROOT(&b->waiters));
 888         WARN_ON(rcu_access_pointer(b->first_signal));
 889         WARN_ON(!RB_EMPTY_ROOT(&b->signals));
 890
 891         if (!IS_ERR_OR_NULL(b->signaler))
 892                 kthread_stop(b->signaler);
 893
 894         cancel_fake_irq(engine);
 895 }
 896
 897 bool intel_breadcrumbs_busy(struct intel_engine_cs *engine)
 898 {
 899         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 900         bool busy = false;
 901
 902         spin_lock_irq(&b->rb_lock);
 903
 904         if (b->irq_wait) {
 905                 wake_up_process(b->irq_wait->tsk);
 906                 busy = true;
 907         }
 908
 909         if (rcu_access_pointer(b->first_signal)) {
 910                 wake_up_process(b->signaler);
 911                 busy = true;
 912         }
 913
 914         spin_unlock_irq(&b->rb_lock);
 915
 916         return busy;
 917 }
 918
 919 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 920 #include "selftests/intel_breadcrumbs.c"
 921 #endif