drivers/gpu/drm/i915/gt/selftest_lrc.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2018 Intel Corporation
   5  */
   6
   7 #include <linux/prime_numbers.h>
   8
   9 #include "gem/i915_gem_pm.h"
  10 #include "gt/intel_reset.h"
  11
  12 #include "i915_selftest.h"
  13 #include "selftests/i915_random.h"
  14 #include "selftests/igt_flush_test.h"
  15 #include "selftests/igt_live_test.h"
  16 #include "selftests/igt_spinner.h"
  17 #include "selftests/lib_sw_fence.h"
  18
  19 #include "gem/selftests/igt_gem_utils.h"
  20 #include "gem/selftests/mock_context.h"
  21
  22 static int live_sanitycheck(void *arg)
  23 {
  24         struct drm_i915_private *i915 = arg;
  25         struct i915_gem_engines_iter it;
  26         struct i915_gem_context *ctx;
  27         struct intel_context *ce;
  28         struct igt_spinner spin;
  29         intel_wakeref_t wakeref;
  30         int err = -ENOMEM;
  31
  32         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
  33                 return 0;
  34
  35         mutex_lock(&i915->drm.struct_mutex);
  36         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
  37
  38         if (igt_spinner_init(&spin, &i915->gt))
  39                 goto err_unlock;
  40
  41         ctx = kernel_context(i915);
  42         if (!ctx)
  43                 goto err_spin;
  44
  45         for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
  46                 struct i915_request *rq;
  47
  48                 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
  49                 if (IS_ERR(rq)) {
  50                         err = PTR_ERR(rq);
  51                         goto err_ctx;
  52                 }
  53
  54                 i915_request_add(rq);
  55                 if (!igt_wait_for_spinner(&spin, rq)) {
  56                         GEM_TRACE("spinner failed to start\n");
  57                         GEM_TRACE_DUMP();
  58                         intel_gt_set_wedged(&i915->gt);
  59                         err = -EIO;
  60                         goto err_ctx;
  61                 }
  62
  63                 igt_spinner_end(&spin);
  64                 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
  65                         err = -EIO;
  66                         goto err_ctx;
  67                 }
  68         }
  69
  70         err = 0;
  71 err_ctx:
  72         i915_gem_context_unlock_engines(ctx);
  73         kernel_context_close(ctx);
  74 err_spin:
  75         igt_spinner_fini(&spin);
  76 err_unlock:
  77         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
  78         mutex_unlock(&i915->drm.struct_mutex);
  79         return err;
  80 }
  81
  82 static int
  83 emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
  84 {
  85         u32 *cs;
  86
  87         cs = intel_ring_begin(rq, 10);
  88         if (IS_ERR(cs))
  89                 return PTR_ERR(cs);
  90
  91         *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
  92
  93         *cs++ = MI_SEMAPHORE_WAIT |
  94                 MI_SEMAPHORE_GLOBAL_GTT |
  95                 MI_SEMAPHORE_POLL |
  96                 MI_SEMAPHORE_SAD_NEQ_SDD;
  97         *cs++ = 0;
  98         *cs++ = i915_ggtt_offset(vma) + 4 * idx;
  99         *cs++ = 0;
 100
 101         if (idx > 0) {
 102                 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 103                 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
 104                 *cs++ = 0;
 105                 *cs++ = 1;
 106         } else {
 107                 *cs++ = MI_NOOP;
 108                 *cs++ = MI_NOOP;
 109                 *cs++ = MI_NOOP;
 110                 *cs++ = MI_NOOP;
 111         }
 112
 113         *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
 114
 115         intel_ring_advance(rq, cs);
 116         return 0;
 117 }
 118
 119 static struct i915_request *
 120 semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
 121 {
 122         struct i915_gem_context *ctx;
 123         struct i915_request *rq;
 124         int err;
 125
 126         ctx = kernel_context(engine->i915);
 127         if (!ctx)
 128                 return ERR_PTR(-ENOMEM);
 129
 130         rq = igt_request_alloc(ctx, engine);
 131         if (IS_ERR(rq))
 132                 goto out_ctx;
 133
 134         err = emit_semaphore_chain(rq, vma, idx);
 135         i915_request_add(rq);
 136         if (err)
 137                 rq = ERR_PTR(err);
 138
 139 out_ctx:
 140         kernel_context_close(ctx);
 141         return rq;
 142 }
 143
 144 static int
 145 release_queue(struct intel_engine_cs *engine,
 146               struct i915_vma *vma,
 147               int idx)
 148 {
 149         struct i915_sched_attr attr = {
 150                 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
 151         };
 152         struct i915_request *rq;
 153         u32 *cs;
 154
 155         rq = i915_request_create(engine->kernel_context);
 156         if (IS_ERR(rq))
 157                 return PTR_ERR(rq);
 158
 159         cs = intel_ring_begin(rq, 4);
 160         if (IS_ERR(cs)) {
 161                 i915_request_add(rq);
 162                 return PTR_ERR(cs);
 163         }
 164
 165         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 166         *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
 167         *cs++ = 0;
 168         *cs++ = 1;
 169
 170         intel_ring_advance(rq, cs);
 171         i915_request_add(rq);
 172
 173         engine->schedule(rq, &attr);
 174
 175         return 0;
 176 }
 177
 178 static int
 179 slice_semaphore_queue(struct intel_engine_cs *outer,
 180                       struct i915_vma *vma,
 181                       int count)
 182 {
 183         struct intel_engine_cs *engine;
 184         struct i915_request *head;
 185         enum intel_engine_id id;
 186         int err, i, n = 0;
 187
 188         head = semaphore_queue(outer, vma, n++);
 189         if (IS_ERR(head))
 190                 return PTR_ERR(head);
 191
 192         i915_request_get(head);
 193         for_each_engine(engine, outer->i915, id) {
 194                 for (i = 0; i < count; i++) {
 195                         struct i915_request *rq;
 196
 197                         rq = semaphore_queue(engine, vma, n++);
 198                         if (IS_ERR(rq)) {
 199                                 err = PTR_ERR(rq);
 200                                 goto out;
 201                         }
 202                 }
 203         }
 204
 205         err = release_queue(outer, vma, n);
 206         if (err)
 207                 goto out;
 208
 209         if (i915_request_wait(head,
 210                               I915_WAIT_LOCKED,
 211                               2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
 212                 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
 213                        count, n);
 214                 GEM_TRACE_DUMP();
 215                 intel_gt_set_wedged(outer->gt);
 216                 err = -EIO;
 217         }
 218
 219 out:
 220         i915_request_put(head);
 221         return err;
 222 }
 223
 224 static int live_timeslice_preempt(void *arg)
 225 {
 226         struct drm_i915_private *i915 = arg;
 227         struct drm_i915_gem_object *obj;
 228         intel_wakeref_t wakeref;
 229         struct i915_vma *vma;
 230         void *vaddr;
 231         int err = 0;
 232         int count;
 233
 234         /*
 235          * If a request takes too long, we would like to give other users
 236          * a fair go on the GPU. In particular, users may create batches
 237          * that wait upon external input, where that input may even be
 238          * supplied by another GPU job. To avoid blocking forever, we
 239          * need to preempt the current task and replace it with another
 240          * ready task.
 241          */
 242
 243         mutex_lock(&i915->drm.struct_mutex);
 244         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 245
 246         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 247         if (IS_ERR(obj)) {
 248                 err = PTR_ERR(obj);
 249                 goto err_unlock;
 250         }
 251
 252         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
 253         if (IS_ERR(vma)) {
 254                 err = PTR_ERR(vma);
 255                 goto err_obj;
 256         }
 257
 258         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
 259         if (IS_ERR(vaddr)) {
 260                 err = PTR_ERR(vaddr);
 261                 goto err_obj;
 262         }
 263
 264         err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 265         if (err)
 266                 goto err_map;
 267
 268         for_each_prime_number_from(count, 1, 16) {
 269                 struct intel_engine_cs *engine;
 270                 enum intel_engine_id id;
 271
 272                 for_each_engine(engine, i915, id) {
 273                         if (!intel_engine_has_preemption(engine))
 274                                 continue;
 275
 276                         memset(vaddr, 0, PAGE_SIZE);
 277
 278                         err = slice_semaphore_queue(engine, vma, count);
 279                         if (err)
 280                                 goto err_pin;
 281
 282                         if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
 283                                 err = -EIO;
 284                                 goto err_pin;
 285                         }
 286                 }
 287         }
 288
 289 err_pin:
 290         i915_vma_unpin(vma);
 291 err_map:
 292         i915_gem_object_unpin_map(obj);
 293 err_obj:
 294         i915_gem_object_put(obj);
 295 err_unlock:
 296         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 297         mutex_unlock(&i915->drm.struct_mutex);
 298
 299         return err;
 300 }
 301
 302 static int live_busywait_preempt(void *arg)
 303 {
 304         struct drm_i915_private *i915 = arg;
 305         struct i915_gem_context *ctx_hi, *ctx_lo;
 306         struct intel_engine_cs *engine;
 307         struct drm_i915_gem_object *obj;
 308         struct i915_vma *vma;
 309         enum intel_engine_id id;
 310         intel_wakeref_t wakeref;
 311         int err = -ENOMEM;
 312         u32 *map;
 313
 314         /*
 315          * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
 316          * preempt the busywaits used to synchronise between rings.
 317          */
 318
 319         mutex_lock(&i915->drm.struct_mutex);
 320         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 321
 322         ctx_hi = kernel_context(i915);
 323         if (!ctx_hi)
 324                 goto err_unlock;
 325         ctx_hi->sched.priority =
 326                 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
 327
 328         ctx_lo = kernel_context(i915);
 329         if (!ctx_lo)
 330                 goto err_ctx_hi;
 331         ctx_lo->sched.priority =
 332                 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
 333
 334         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 335         if (IS_ERR(obj)) {
 336                 err = PTR_ERR(obj);
 337                 goto err_ctx_lo;
 338         }
 339
 340         map = i915_gem_object_pin_map(obj, I915_MAP_WC);
 341         if (IS_ERR(map)) {
 342                 err = PTR_ERR(map);
 343                 goto err_obj;
 344         }
 345
 346         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
 347         if (IS_ERR(vma)) {
 348                 err = PTR_ERR(vma);
 349                 goto err_map;
 350         }
 351
 352         err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 353         if (err)
 354                 goto err_map;
 355
 356         for_each_engine(engine, i915, id) {
 357                 struct i915_request *lo, *hi;
 358                 struct igt_live_test t;
 359                 u32 *cs;
 360
 361                 if (!intel_engine_has_preemption(engine))
 362                         continue;
 363
 364                 if (!intel_engine_can_store_dword(engine))
 365                         continue;
 366
 367                 if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
 368                         err = -EIO;
 369                         goto err_vma;
 370                 }
 371
 372                 /*
 373                  * We create two requests. The low priority request
 374                  * busywaits on a semaphore (inside the ringbuffer where
 375                  * is should be preemptible) and the high priority requests
 376                  * uses a MI_STORE_DWORD_IMM to update the semaphore value
 377                  * allowing the first request to complete. If preemption
 378                  * fails, we hang instead.
 379                  */
 380
 381                 lo = igt_request_alloc(ctx_lo, engine);
 382                 if (IS_ERR(lo)) {
 383                         err = PTR_ERR(lo);
 384                         goto err_vma;
 385                 }
 386
 387                 cs = intel_ring_begin(lo, 8);
 388                 if (IS_ERR(cs)) {
 389                         err = PTR_ERR(cs);
 390                         i915_request_add(lo);
 391                         goto err_vma;
 392                 }
 393
 394                 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 395                 *cs++ = i915_ggtt_offset(vma);
 396                 *cs++ = 0;
 397                 *cs++ = 1;
 398
 399                 /* XXX Do we need a flush + invalidate here? */
 400
 401                 *cs++ = MI_SEMAPHORE_WAIT |
 402                         MI_SEMAPHORE_GLOBAL_GTT |
 403                         MI_SEMAPHORE_POLL |
 404                         MI_SEMAPHORE_SAD_EQ_SDD;
 405                 *cs++ = 0;
 406                 *cs++ = i915_ggtt_offset(vma);
 407                 *cs++ = 0;
 408
 409                 intel_ring_advance(lo, cs);
 410                 i915_request_add(lo);
 411
 412                 if (wait_for(READ_ONCE(*map), 10)) {
 413                         err = -ETIMEDOUT;
 414                         goto err_vma;
 415                 }
 416
 417                 /* Low priority request should be busywaiting now */
 418                 if (i915_request_wait(lo, 0, 1) != -ETIME) {
 419                         pr_err("%s: Busywaiting request did not!\n",
 420                                engine->name);
 421                         err = -EIO;
 422                         goto err_vma;
 423                 }
 424
 425                 hi = igt_request_alloc(ctx_hi, engine);
 426                 if (IS_ERR(hi)) {
 427                         err = PTR_ERR(hi);
 428                         goto err_vma;
 429                 }
 430
 431                 cs = intel_ring_begin(hi, 4);
 432                 if (IS_ERR(cs)) {
 433                         err = PTR_ERR(cs);
 434                         i915_request_add(hi);
 435                         goto err_vma;
 436                 }
 437
 438                 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
 439                 *cs++ = i915_ggtt_offset(vma);
 440                 *cs++ = 0;
 441                 *cs++ = 0;
 442
 443                 intel_ring_advance(hi, cs);
 444                 i915_request_add(hi);
 445
 446                 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
 447                         struct drm_printer p = drm_info_printer(i915->drm.dev);
 448
 449                         pr_err("%s: Failed to preempt semaphore busywait!\n",
 450                                engine->name);
 451
 452                         intel_engine_dump(engine, &p, "%s\n", engine->name);
 453                         GEM_TRACE_DUMP();
 454
 455                         intel_gt_set_wedged(&i915->gt);
 456                         err = -EIO;
 457                         goto err_vma;
 458                 }
 459                 GEM_BUG_ON(READ_ONCE(*map));
 460
 461                 if (igt_live_test_end(&t)) {
 462                         err = -EIO;
 463                         goto err_vma;
 464                 }
 465         }
 466
 467         err = 0;
 468 err_vma:
 469         i915_vma_unpin(vma);
 470 err_map:
 471         i915_gem_object_unpin_map(obj);
 472 err_obj:
 473         i915_gem_object_put(obj);
 474 err_ctx_lo:
 475         kernel_context_close(ctx_lo);
 476 err_ctx_hi:
 477         kernel_context_close(ctx_hi);
 478 err_unlock:
 479         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 480         mutex_unlock(&i915->drm.struct_mutex);
 481         return err;
 482 }
 483
 484 static struct i915_request *
 485 spinner_create_request(struct igt_spinner *spin,
 486                        struct i915_gem_context *ctx,
 487                        struct intel_engine_cs *engine,
 488                        u32 arb)
 489 {
 490         struct intel_context *ce;
 491         struct i915_request *rq;
 492
 493         ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
 494         if (IS_ERR(ce))
 495                 return ERR_CAST(ce);
 496
 497         rq = igt_spinner_create_request(spin, ce, arb);
 498         intel_context_put(ce);
 499         return rq;
 500 }
 501
 502 static int live_preempt(void *arg)
 503 {
 504         struct drm_i915_private *i915 = arg;
 505         struct i915_gem_context *ctx_hi, *ctx_lo;
 506         struct igt_spinner spin_hi, spin_lo;
 507         struct intel_engine_cs *engine;
 508         enum intel_engine_id id;
 509         intel_wakeref_t wakeref;
 510         int err = -ENOMEM;
 511
 512         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
 513                 return 0;
 514
 515         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
 516                 pr_err("Logical preemption supported, but not exposed\n");
 517
 518         mutex_lock(&i915->drm.struct_mutex);
 519         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 520
 521         if (igt_spinner_init(&spin_hi, &i915->gt))
 522                 goto err_unlock;
 523
 524         if (igt_spinner_init(&spin_lo, &i915->gt))
 525                 goto err_spin_hi;
 526
 527         ctx_hi = kernel_context(i915);
 528         if (!ctx_hi)
 529                 goto err_spin_lo;
 530         ctx_hi->sched.priority =
 531                 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
 532
 533         ctx_lo = kernel_context(i915);
 534         if (!ctx_lo)
 535                 goto err_ctx_hi;
 536         ctx_lo->sched.priority =
 537                 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
 538
 539         for_each_engine(engine, i915, id) {
 540                 struct igt_live_test t;
 541                 struct i915_request *rq;
 542
 543                 if (!intel_engine_has_preemption(engine))
 544                         continue;
 545
 546                 if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
 547                         err = -EIO;
 548                         goto err_ctx_lo;
 549                 }
 550
 551                 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
 552                                             MI_ARB_CHECK);
 553                 if (IS_ERR(rq)) {
 554                         err = PTR_ERR(rq);
 555                         goto err_ctx_lo;
 556                 }
 557
 558                 i915_request_add(rq);
 559                 if (!igt_wait_for_spinner(&spin_lo, rq)) {
 560                         GEM_TRACE("lo spinner failed to start\n");
 561                         GEM_TRACE_DUMP();
 562                         intel_gt_set_wedged(&i915->gt);
 563                         err = -EIO;
 564                         goto err_ctx_lo;
 565                 }
 566
 567                 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
 568                                             MI_ARB_CHECK);
 569                 if (IS_ERR(rq)) {
 570                         igt_spinner_end(&spin_lo);
 571                         err = PTR_ERR(rq);
 572                         goto err_ctx_lo;
 573                 }
 574
 575                 i915_request_add(rq);
 576                 if (!igt_wait_for_spinner(&spin_hi, rq)) {
 577                         GEM_TRACE("hi spinner failed to start\n");
 578                         GEM_TRACE_DUMP();
 579                         intel_gt_set_wedged(&i915->gt);
 580                         err = -EIO;
 581                         goto err_ctx_lo;
 582                 }
 583
 584                 igt_spinner_end(&spin_hi);
 585                 igt_spinner_end(&spin_lo);
 586
 587                 if (igt_live_test_end(&t)) {
 588                         err = -EIO;
 589                         goto err_ctx_lo;
 590                 }
 591         }
 592
 593         err = 0;
 594 err_ctx_lo:
 595         kernel_context_close(ctx_lo);
 596 err_ctx_hi:
 597         kernel_context_close(ctx_hi);
 598 err_spin_lo:
 599         igt_spinner_fini(&spin_lo);
 600 err_spin_hi:
 601         igt_spinner_fini(&spin_hi);
 602 err_unlock:
 603         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 604         mutex_unlock(&i915->drm.struct_mutex);
 605         return err;
 606 }
 607
 608 static int live_late_preempt(void *arg)
 609 {
 610         struct drm_i915_private *i915 = arg;
 611         struct i915_gem_context *ctx_hi, *ctx_lo;
 612         struct igt_spinner spin_hi, spin_lo;
 613         struct intel_engine_cs *engine;
 614         struct i915_sched_attr attr = {};
 615         enum intel_engine_id id;
 616         intel_wakeref_t wakeref;
 617         int err = -ENOMEM;
 618
 619         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
 620                 return 0;
 621
 622         mutex_lock(&i915->drm.struct_mutex);
 623         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 624
 625         if (igt_spinner_init(&spin_hi, &i915->gt))
 626                 goto err_unlock;
 627
 628         if (igt_spinner_init(&spin_lo, &i915->gt))
 629                 goto err_spin_hi;
 630
 631         ctx_hi = kernel_context(i915);
 632         if (!ctx_hi)
 633                 goto err_spin_lo;
 634
 635         ctx_lo = kernel_context(i915);
 636         if (!ctx_lo)
 637                 goto err_ctx_hi;
 638
 639         /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
 640         ctx_lo->sched.priority = I915_USER_PRIORITY(1);
 641
 642         for_each_engine(engine, i915, id) {
 643                 struct igt_live_test t;
 644                 struct i915_request *rq;
 645
 646                 if (!intel_engine_has_preemption(engine))
 647                         continue;
 648
 649                 if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
 650                         err = -EIO;
 651                         goto err_ctx_lo;
 652                 }
 653
 654                 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
 655                                             MI_ARB_CHECK);
 656                 if (IS_ERR(rq)) {
 657                         err = PTR_ERR(rq);
 658                         goto err_ctx_lo;
 659                 }
 660
 661                 i915_request_add(rq);
 662                 if (!igt_wait_for_spinner(&spin_lo, rq)) {
 663                         pr_err("First context failed to start\n");
 664                         goto err_wedged;
 665                 }
 666
 667                 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
 668                                             MI_NOOP);
 669                 if (IS_ERR(rq)) {
 670                         igt_spinner_end(&spin_lo);
 671                         err = PTR_ERR(rq);
 672                         goto err_ctx_lo;
 673                 }
 674
 675                 i915_request_add(rq);
 676                 if (igt_wait_for_spinner(&spin_hi, rq)) {
 677                         pr_err("Second context overtook first?\n");
 678                         goto err_wedged;
 679                 }
 680
 681                 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
 682                 engine->schedule(rq, &attr);
 683
 684                 if (!igt_wait_for_spinner(&spin_hi, rq)) {
 685                         pr_err("High priority context failed to preempt the low priority context\n");
 686                         GEM_TRACE_DUMP();
 687                         goto err_wedged;
 688                 }
 689
 690                 igt_spinner_end(&spin_hi);
 691                 igt_spinner_end(&spin_lo);
 692
 693                 if (igt_live_test_end(&t)) {
 694                         err = -EIO;
 695                         goto err_ctx_lo;
 696                 }
 697         }
 698
 699         err = 0;
 700 err_ctx_lo:
 701         kernel_context_close(ctx_lo);
 702 err_ctx_hi:
 703         kernel_context_close(ctx_hi);
 704 err_spin_lo:
 705         igt_spinner_fini(&spin_lo);
 706 err_spin_hi:
 707         igt_spinner_fini(&spin_hi);
 708 err_unlock:
 709         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 710         mutex_unlock(&i915->drm.struct_mutex);
 711         return err;
 712
 713 err_wedged:
 714         igt_spinner_end(&spin_hi);
 715         igt_spinner_end(&spin_lo);
 716         intel_gt_set_wedged(&i915->gt);
 717         err = -EIO;
 718         goto err_ctx_lo;
 719 }
 720
 721 struct preempt_client {
 722         struct igt_spinner spin;
 723         struct i915_gem_context *ctx;
 724 };
 725
 726 static int preempt_client_init(struct drm_i915_private *i915,
 727                                struct preempt_client *c)
 728 {
 729         c->ctx = kernel_context(i915);
 730         if (!c->ctx)
 731                 return -ENOMEM;
 732
 733         if (igt_spinner_init(&c->spin, &i915->gt))
 734                 goto err_ctx;
 735
 736         return 0;
 737
 738 err_ctx:
 739         kernel_context_close(c->ctx);
 740         return -ENOMEM;
 741 }
 742
 743 static void preempt_client_fini(struct preempt_client *c)
 744 {
 745         igt_spinner_fini(&c->spin);
 746         kernel_context_close(c->ctx);
 747 }
 748
 749 static int live_nopreempt(void *arg)
 750 {
 751         struct drm_i915_private *i915 = arg;
 752         struct intel_engine_cs *engine;
 753         struct preempt_client a, b;
 754         enum intel_engine_id id;
 755         intel_wakeref_t wakeref;
 756         int err = -ENOMEM;
 757
 758         /*
 759          * Verify that we can disable preemption for an individual request
 760          * that may be being observed and not want to be interrupted.
 761          */
 762
 763         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
 764                 return 0;
 765
 766         mutex_lock(&i915->drm.struct_mutex);
 767         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 768
 769         if (preempt_client_init(i915, &a))
 770                 goto err_unlock;
 771         if (preempt_client_init(i915, &b))
 772                 goto err_client_a;
 773         b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
 774
 775         for_each_engine(engine, i915, id) {
 776                 struct i915_request *rq_a, *rq_b;
 777
 778                 if (!intel_engine_has_preemption(engine))
 779                         continue;
 780
 781                 engine->execlists.preempt_hang.count = 0;
 782
 783                 rq_a = spinner_create_request(&a.spin,
 784                                               a.ctx, engine,
 785                                               MI_ARB_CHECK);
 786                 if (IS_ERR(rq_a)) {
 787                         err = PTR_ERR(rq_a);
 788                         goto err_client_b;
 789                 }
 790
 791                 /* Low priority client, but unpreemptable! */
 792                 rq_a->flags |= I915_REQUEST_NOPREEMPT;
 793
 794                 i915_request_add(rq_a);
 795                 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
 796                         pr_err("First client failed to start\n");
 797                         goto err_wedged;
 798                 }
 799
 800                 rq_b = spinner_create_request(&b.spin,
 801                                               b.ctx, engine,
 802                                               MI_ARB_CHECK);
 803                 if (IS_ERR(rq_b)) {
 804                         err = PTR_ERR(rq_b);
 805                         goto err_client_b;
 806                 }
 807
 808                 i915_request_add(rq_b);
 809
 810                 /* B is much more important than A! (But A is unpreemptable.) */
 811                 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
 812
 813                 /* Wait long enough for preemption and timeslicing */
 814                 if (igt_wait_for_spinner(&b.spin, rq_b)) {
 815                         pr_err("Second client started too early!\n");
 816                         goto err_wedged;
 817                 }
 818
 819                 igt_spinner_end(&a.spin);
 820
 821                 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
 822                         pr_err("Second client failed to start\n");
 823                         goto err_wedged;
 824                 }
 825
 826                 igt_spinner_end(&b.spin);
 827
 828                 if (engine->execlists.preempt_hang.count) {
 829                         pr_err("Preemption recorded x%d; should have been suppressed!\n",
 830                                engine->execlists.preempt_hang.count);
 831                         err = -EINVAL;
 832                         goto err_wedged;
 833                 }
 834
 835                 if (igt_flush_test(i915, I915_WAIT_LOCKED))
 836                         goto err_wedged;
 837         }
 838
 839         err = 0;
 840 err_client_b:
 841         preempt_client_fini(&b);
 842 err_client_a:
 843         preempt_client_fini(&a);
 844 err_unlock:
 845         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 846         mutex_unlock(&i915->drm.struct_mutex);
 847         return err;
 848
 849 err_wedged:
 850         igt_spinner_end(&b.spin);
 851         igt_spinner_end(&a.spin);
 852         intel_gt_set_wedged(&i915->gt);
 853         err = -EIO;
 854         goto err_client_b;
 855 }
 856
 857 static int live_suppress_self_preempt(void *arg)
 858 {
 859         struct drm_i915_private *i915 = arg;
 860         struct intel_engine_cs *engine;
 861         struct i915_sched_attr attr = {
 862                 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
 863         };
 864         struct preempt_client a, b;
 865         enum intel_engine_id id;
 866         intel_wakeref_t wakeref;
 867         int err = -ENOMEM;
 868
 869         /*
 870          * Verify that if a preemption request does not cause a change in
 871          * the current execution order, the preempt-to-idle injection is
 872          * skipped and that we do not accidentally apply it after the CS
 873          * completion event.
 874          */
 875
 876         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
 877                 return 0;
 878
 879         if (USES_GUC_SUBMISSION(i915))
 880                 return 0; /* presume black blox */
 881
 882         if (intel_vgpu_active(i915))
 883                 return 0; /* GVT forces single port & request submission */
 884
 885         mutex_lock(&i915->drm.struct_mutex);
 886         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 887
 888         if (preempt_client_init(i915, &a))
 889                 goto err_unlock;
 890         if (preempt_client_init(i915, &b))
 891                 goto err_client_a;
 892
 893         for_each_engine(engine, i915, id) {
 894                 struct i915_request *rq_a, *rq_b;
 895                 int depth;
 896
 897                 if (!intel_engine_has_preemption(engine))
 898                         continue;
 899
 900                 if (igt_flush_test(i915, I915_WAIT_LOCKED))
 901                         goto err_wedged;
 902
 903                 intel_engine_pm_get(engine);
 904                 engine->execlists.preempt_hang.count = 0;
 905
 906                 rq_a = spinner_create_request(&a.spin,
 907                                               a.ctx, engine,
 908                                               MI_NOOP);
 909                 if (IS_ERR(rq_a)) {
 910                         err = PTR_ERR(rq_a);
 911                         intel_engine_pm_put(engine);
 912                         goto err_client_b;
 913                 }
 914
 915                 i915_request_add(rq_a);
 916                 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
 917                         pr_err("First client failed to start\n");
 918                         intel_engine_pm_put(engine);
 919                         goto err_wedged;
 920                 }
 921
 922                 /* Keep postponing the timer to avoid premature slicing */
 923                 mod_timer(&engine->execlists.timer, jiffies + HZ);
 924                 for (depth = 0; depth < 8; depth++) {
 925                         rq_b = spinner_create_request(&b.spin,
 926                                                       b.ctx, engine,
 927                                                       MI_NOOP);
 928                         if (IS_ERR(rq_b)) {
 929                                 err = PTR_ERR(rq_b);
 930                                 intel_engine_pm_put(engine);
 931                                 goto err_client_b;
 932                         }
 933                         i915_request_add(rq_b);
 934
 935                         GEM_BUG_ON(i915_request_completed(rq_a));
 936                         engine->schedule(rq_a, &attr);
 937                         igt_spinner_end(&a.spin);
 938
 939                         if (!igt_wait_for_spinner(&b.spin, rq_b)) {
 940                                 pr_err("Second client failed to start\n");
 941                                 intel_engine_pm_put(engine);
 942                                 goto err_wedged;
 943                         }
 944
 945                         swap(a, b);
 946                         rq_a = rq_b;
 947                 }
 948                 igt_spinner_end(&a.spin);
 949
 950                 if (engine->execlists.preempt_hang.count) {
 951                         pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
 952                                engine->name,
 953                                engine->execlists.preempt_hang.count,
 954                                depth);
 955                         intel_engine_pm_put(engine);
 956                         err = -EINVAL;
 957                         goto err_client_b;
 958                 }
 959
 960                 intel_engine_pm_put(engine);
 961                 if (igt_flush_test(i915, I915_WAIT_LOCKED))
 962                         goto err_wedged;
 963         }
 964
 965         err = 0;
 966 err_client_b:
 967         preempt_client_fini(&b);
 968 err_client_a:
 969         preempt_client_fini(&a);
 970 err_unlock:
 971         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 972         mutex_unlock(&i915->drm.struct_mutex);
 973         return err;
 974
 975 err_wedged:
 976         igt_spinner_end(&b.spin);
 977         igt_spinner_end(&a.spin);
 978         intel_gt_set_wedged(&i915->gt);
 979         err = -EIO;
 980         goto err_client_b;
 981 }
 982
 983 static int __i915_sw_fence_call
 984 dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 985 {
 986         return NOTIFY_DONE;
 987 }
 988
 989 static struct i915_request *dummy_request(struct intel_engine_cs *engine)
 990 {
 991         struct i915_request *rq;
 992
 993         rq = kzalloc(sizeof(*rq), GFP_KERNEL);
 994         if (!rq)
 995                 return NULL;
 996
 997         INIT_LIST_HEAD(&rq->active_list);
 998         rq->engine = engine;
 999
1000         i915_sched_node_init(&rq->sched);
1001
1002         /* mark this request as permanently incomplete */
1003         rq->fence.seqno = 1;
1004         BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
1005         rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
1006         GEM_BUG_ON(i915_request_completed(rq));
1007
1008         i915_sw_fence_init(&rq->submit, dummy_notify);
1009         set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
1010
1011         spin_lock_init(&rq->lock);
1012         rq->fence.lock = &rq->lock;
1013         INIT_LIST_HEAD(&rq->fence.cb_list);
1014
1015         return rq;
1016 }
1017
1018 static void dummy_request_free(struct i915_request *dummy)
1019 {
1020         /* We have to fake the CS interrupt to kick the next request */
1021         i915_sw_fence_commit(&dummy->submit);
1022
1023         i915_request_mark_complete(dummy);
1024         dma_fence_signal(&dummy->fence);
1025
1026         i915_sched_node_fini(&dummy->sched);
1027         i915_sw_fence_fini(&dummy->submit);
1028
1029         dma_fence_free(&dummy->fence);
1030 }
1031
1032 static int live_suppress_wait_preempt(void *arg)
1033 {
1034         struct drm_i915_private *i915 = arg;
1035         struct preempt_client client[4];
1036         struct intel_engine_cs *engine;
1037         enum intel_engine_id id;
1038         intel_wakeref_t wakeref;
1039         int err = -ENOMEM;
1040         int i;
1041
1042         /*
1043          * Waiters are given a little priority nudge, but not enough
1044          * to actually cause any preemption. Double check that we do
1045          * not needlessly generate preempt-to-idle cycles.
1046          */
1047
1048         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
1049                 return 0;
1050
1051         mutex_lock(&i915->drm.struct_mutex);
1052         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1053
1054         if (preempt_client_init(i915, &client[0])) /* ELSP[0] */
1055                 goto err_unlock;
1056         if (preempt_client_init(i915, &client[1])) /* ELSP[1] */
1057                 goto err_client_0;
1058         if (preempt_client_init(i915, &client[2])) /* head of queue */
1059                 goto err_client_1;
1060         if (preempt_client_init(i915, &client[3])) /* bystander */
1061                 goto err_client_2;
1062
1063         for_each_engine(engine, i915, id) {
1064                 int depth;
1065
1066                 if (!intel_engine_has_preemption(engine))
1067                         continue;
1068
1069                 if (!engine->emit_init_breadcrumb)
1070                         continue;
1071
1072                 for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
1073                         struct i915_request *rq[ARRAY_SIZE(client)];
1074                         struct i915_request *dummy;
1075
1076                         engine->execlists.preempt_hang.count = 0;
1077
1078                         dummy = dummy_request(engine);
1079                         if (!dummy)
1080                                 goto err_client_3;
1081
1082                         for (i = 0; i < ARRAY_SIZE(client); i++) {
1083                                 rq[i] = spinner_create_request(&client[i].spin,
1084                                                                client[i].ctx, engine,
1085                                                                MI_NOOP);
1086                                 if (IS_ERR(rq[i])) {
1087                                         err = PTR_ERR(rq[i]);
1088                                         goto err_wedged;
1089                                 }
1090
1091                                 /* Disable NEWCLIENT promotion */
1092                                 __i915_active_request_set(&i915_request_timeline(rq[i])->last_request,
1093                                                           dummy);
1094                                 i915_request_add(rq[i]);
1095                         }
1096
1097                         dummy_request_free(dummy);
1098
1099                         GEM_BUG_ON(i915_request_completed(rq[0]));
1100                         if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
1101                                 pr_err("%s: First client failed to start\n",
1102                                        engine->name);
1103                                 goto err_wedged;
1104                         }
1105                         GEM_BUG_ON(!i915_request_started(rq[0]));
1106
1107                         if (i915_request_wait(rq[depth],
1108                                               I915_WAIT_PRIORITY,
1109                                               1) != -ETIME) {
1110                                 pr_err("%s: Waiter depth:%d completed!\n",
1111                                        engine->name, depth);
1112                                 goto err_wedged;
1113                         }
1114
1115                         for (i = 0; i < ARRAY_SIZE(client); i++)
1116                                 igt_spinner_end(&client[i].spin);
1117
1118                         if (igt_flush_test(i915, I915_WAIT_LOCKED))
1119                                 goto err_wedged;
1120
1121                         if (engine->execlists.preempt_hang.count) {
1122                                 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
1123                                        engine->name,
1124                                        engine->execlists.preempt_hang.count,
1125                                        depth);
1126                                 err = -EINVAL;
1127                                 goto err_client_3;
1128                         }
1129                 }
1130         }
1131
1132         err = 0;
1133 err_client_3:
1134         preempt_client_fini(&client[3]);
1135 err_client_2:
1136         preempt_client_fini(&client[2]);
1137 err_client_1:
1138         preempt_client_fini(&client[1]);
1139 err_client_0:
1140         preempt_client_fini(&client[0]);
1141 err_unlock:
1142         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1143         mutex_unlock(&i915->drm.struct_mutex);
1144         return err;
1145
1146 err_wedged:
1147         for (i = 0; i < ARRAY_SIZE(client); i++)
1148                 igt_spinner_end(&client[i].spin);
1149         intel_gt_set_wedged(&i915->gt);
1150         err = -EIO;
1151         goto err_client_3;
1152 }
1153
1154 static int live_chain_preempt(void *arg)
1155 {
1156         struct drm_i915_private *i915 = arg;
1157         struct intel_engine_cs *engine;
1158         struct preempt_client hi, lo;
1159         enum intel_engine_id id;
1160         intel_wakeref_t wakeref;
1161         int err = -ENOMEM;
1162
1163         /*
1164          * Build a chain AB...BA between two contexts (A, B) and request
1165          * preemption of the last request. It should then complete before
1166          * the previously submitted spinner in B.
1167          */
1168
1169         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
1170                 return 0;
1171
1172         mutex_lock(&i915->drm.struct_mutex);
1173         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1174
1175         if (preempt_client_init(i915, &hi))
1176                 goto err_unlock;
1177
1178         if (preempt_client_init(i915, &lo))
1179                 goto err_client_hi;
1180
1181         for_each_engine(engine, i915, id) {
1182                 struct i915_sched_attr attr = {
1183                         .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1184                 };
1185                 struct igt_live_test t;
1186                 struct i915_request *rq;
1187                 int ring_size, count, i;
1188
1189                 if (!intel_engine_has_preemption(engine))
1190                         continue;
1191
1192                 rq = spinner_create_request(&lo.spin,
1193                                             lo.ctx, engine,
1194                                             MI_ARB_CHECK);
1195                 if (IS_ERR(rq))
1196                         goto err_wedged;
1197                 i915_request_add(rq);
1198
1199                 ring_size = rq->wa_tail - rq->head;
1200                 if (ring_size < 0)
1201                         ring_size += rq->ring->size;
1202                 ring_size = rq->ring->size / ring_size;
1203                 pr_debug("%s(%s): Using maximum of %d requests\n",
1204                          __func__, engine->name, ring_size);
1205
1206                 igt_spinner_end(&lo.spin);
1207                 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1208                         pr_err("Timed out waiting to flush %s\n", engine->name);
1209                         goto err_wedged;
1210                 }
1211
1212                 if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
1213                         err = -EIO;
1214                         goto err_wedged;
1215                 }
1216
1217                 for_each_prime_number_from(count, 1, ring_size) {
1218                         rq = spinner_create_request(&hi.spin,
1219                                                     hi.ctx, engine,
1220                                                     MI_ARB_CHECK);
1221                         if (IS_ERR(rq))
1222                                 goto err_wedged;
1223                         i915_request_add(rq);
1224                         if (!igt_wait_for_spinner(&hi.spin, rq))
1225                                 goto err_wedged;
1226
1227                         rq = spinner_create_request(&lo.spin,
1228                                                     lo.ctx, engine,
1229                                                     MI_ARB_CHECK);
1230                         if (IS_ERR(rq))
1231                                 goto err_wedged;
1232                         i915_request_add(rq);
1233
1234                         for (i = 0; i < count; i++) {
1235                                 rq = igt_request_alloc(lo.ctx, engine);
1236                                 if (IS_ERR(rq))
1237                                         goto err_wedged;
1238                                 i915_request_add(rq);
1239                         }
1240
1241                         rq = igt_request_alloc(hi.ctx, engine);
1242                         if (IS_ERR(rq))
1243                                 goto err_wedged;
1244                         i915_request_add(rq);
1245                         engine->schedule(rq, &attr);
1246
1247                         igt_spinner_end(&hi.spin);
1248                         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1249                                 struct drm_printer p =
1250                                         drm_info_printer(i915->drm.dev);
1251
1252                                 pr_err("Failed to preempt over chain of %d\n",
1253                                        count);
1254                                 intel_engine_dump(engine, &p,
1255                                                   "%s\n", engine->name);
1256                                 goto err_wedged;
1257                         }
1258                         igt_spinner_end(&lo.spin);
1259
1260                         rq = igt_request_alloc(lo.ctx, engine);
1261                         if (IS_ERR(rq))
1262                                 goto err_wedged;
1263                         i915_request_add(rq);
1264                         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1265                                 struct drm_printer p =
1266                                         drm_info_printer(i915->drm.dev);
1267
1268                                 pr_err("Failed to flush low priority chain of %d requests\n",
1269                                        count);
1270                                 intel_engine_dump(engine, &p,
1271                                                   "%s\n", engine->name);
1272                                 goto err_wedged;
1273                         }
1274                 }
1275
1276                 if (igt_live_test_end(&t)) {
1277                         err = -EIO;
1278                         goto err_wedged;
1279                 }
1280         }
1281
1282         err = 0;
1283 err_client_lo:
1284         preempt_client_fini(&lo);
1285 err_client_hi:
1286         preempt_client_fini(&hi);
1287 err_unlock:
1288         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1289         mutex_unlock(&i915->drm.struct_mutex);
1290         return err;
1291
1292 err_wedged:
1293         igt_spinner_end(&hi.spin);
1294         igt_spinner_end(&lo.spin);
1295         intel_gt_set_wedged(&i915->gt);
1296         err = -EIO;
1297         goto err_client_lo;
1298 }
1299
1300 static int live_preempt_hang(void *arg)
1301 {
1302         struct drm_i915_private *i915 = arg;
1303         struct i915_gem_context *ctx_hi, *ctx_lo;
1304         struct igt_spinner spin_hi, spin_lo;
1305         struct intel_engine_cs *engine;
1306         enum intel_engine_id id;
1307         intel_wakeref_t wakeref;
1308         int err = -ENOMEM;
1309
1310         if (!HAS_LOGICAL_RING_PREEMPTION(i915))
1311                 return 0;
1312
1313         if (!intel_has_reset_engine(&i915->gt))
1314                 return 0;
1315
1316         mutex_lock(&i915->drm.struct_mutex);
1317         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1318
1319         if (igt_spinner_init(&spin_hi, &i915->gt))
1320                 goto err_unlock;
1321
1322         if (igt_spinner_init(&spin_lo, &i915->gt))
1323                 goto err_spin_hi;
1324
1325         ctx_hi = kernel_context(i915);
1326         if (!ctx_hi)
1327                 goto err_spin_lo;
1328         ctx_hi->sched.priority =
1329                 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
1330
1331         ctx_lo = kernel_context(i915);
1332         if (!ctx_lo)
1333                 goto err_ctx_hi;
1334         ctx_lo->sched.priority =
1335                 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
1336
1337         for_each_engine(engine, i915, id) {
1338                 struct i915_request *rq;
1339
1340                 if (!intel_engine_has_preemption(engine))
1341                         continue;
1342
1343                 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1344                                             MI_ARB_CHECK);
1345                 if (IS_ERR(rq)) {
1346                         err = PTR_ERR(rq);
1347                         goto err_ctx_lo;
1348                 }
1349
1350                 i915_request_add(rq);
1351                 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1352                         GEM_TRACE("lo spinner failed to start\n");
1353                         GEM_TRACE_DUMP();
1354                         intel_gt_set_wedged(&i915->gt);
1355                         err = -EIO;
1356                         goto err_ctx_lo;
1357                 }
1358
1359                 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1360                                             MI_ARB_CHECK);
1361                 if (IS_ERR(rq)) {
1362                         igt_spinner_end(&spin_lo);
1363                         err = PTR_ERR(rq);
1364                         goto err_ctx_lo;
1365                 }
1366
1367                 init_completion(&engine->execlists.preempt_hang.completion);
1368                 engine->execlists.preempt_hang.inject_hang = true;
1369
1370                 i915_request_add(rq);
1371
1372                 if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion,
1373                                                  HZ / 10)) {
1374                         pr_err("Preemption did not occur within timeout!");
1375                         GEM_TRACE_DUMP();
1376                         intel_gt_set_wedged(&i915->gt);
1377                         err = -EIO;
1378                         goto err_ctx_lo;
1379                 }
1380
1381                 set_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags);
1382                 intel_engine_reset(engine, NULL);
1383                 clear_bit(I915_RESET_ENGINE + id, &i915->gt.reset.flags);
1384
1385                 engine->execlists.preempt_hang.inject_hang = false;
1386
1387                 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1388                         GEM_TRACE("hi spinner failed to start\n");
1389                         GEM_TRACE_DUMP();
1390                         intel_gt_set_wedged(&i915->gt);
1391                         err = -EIO;
1392                         goto err_ctx_lo;
1393                 }
1394
1395                 igt_spinner_end(&spin_hi);
1396                 igt_spinner_end(&spin_lo);
1397                 if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
1398                         err = -EIO;
1399                         goto err_ctx_lo;
1400                 }
1401         }
1402
1403         err = 0;
1404 err_ctx_lo:
1405         kernel_context_close(ctx_lo);
1406 err_ctx_hi:
1407         kernel_context_close(ctx_hi);
1408 err_spin_lo:
1409         igt_spinner_fini(&spin_lo);
1410 err_spin_hi:
1411         igt_spinner_fini(&spin_hi);
1412 err_unlock:
1413         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1414         mutex_unlock(&i915->drm.struct_mutex);
1415         return err;
1416 }
1417
1418 static int random_range(struct rnd_state *rnd, int min, int max)
1419 {
1420         return i915_prandom_u32_max_state(max - min, rnd) + min;
1421 }
1422
1423 static int random_priority(struct rnd_state *rnd)
1424 {
1425         return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
1426 }
1427
1428 struct preempt_smoke {
1429         struct drm_i915_private *i915;
1430         struct i915_gem_context **contexts;
1431         struct intel_engine_cs *engine;
1432         struct drm_i915_gem_object *batch;
1433         unsigned int ncontext;
1434         struct rnd_state prng;
1435         unsigned long count;
1436 };
1437
1438 static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
1439 {
1440         return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
1441                                                           &smoke->prng)];
1442 }
1443
1444 static int smoke_submit(struct preempt_smoke *smoke,
1445                         struct i915_gem_context *ctx, int prio,
1446                         struct drm_i915_gem_object *batch)
1447 {
1448         struct i915_request *rq;
1449         struct i915_vma *vma = NULL;
1450         int err = 0;
1451
1452         if (batch) {
1453                 vma = i915_vma_instance(batch, ctx->vm, NULL);
1454                 if (IS_ERR(vma))
1455                         return PTR_ERR(vma);
1456
1457                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
1458                 if (err)
1459                         return err;
1460         }
1461
1462         ctx->sched.priority = prio;
1463
1464         rq = igt_request_alloc(ctx, smoke->engine);
1465         if (IS_ERR(rq)) {
1466                 err = PTR_ERR(rq);
1467                 goto unpin;
1468         }
1469
1470         if (vma) {
1471                 i915_vma_lock(vma);
1472                 err = i915_request_await_object(rq, vma->obj, false);
1473                 if (!err)
1474                         err = i915_vma_move_to_active(vma, rq, 0);
1475                 if (!err)
1476                         err = rq->engine->emit_bb_start(rq,
1477                                                         vma->node.start,
1478                                                         PAGE_SIZE, 0);
1479                 i915_vma_unlock(vma);
1480         }
1481
1482         i915_request_add(rq);
1483
1484 unpin:
1485         if (vma)
1486                 i915_vma_unpin(vma);
1487
1488         return err;
1489 }
1490
1491 static int smoke_crescendo_thread(void *arg)
1492 {
1493         struct preempt_smoke *smoke = arg;
1494         IGT_TIMEOUT(end_time);
1495         unsigned long count;
1496
1497         count = 0;
1498         do {
1499                 struct i915_gem_context *ctx = smoke_context(smoke);
1500                 int err;
1501
1502                 mutex_lock(&smoke->i915->drm.struct_mutex);
1503                 err = smoke_submit(smoke,
1504                                    ctx, count % I915_PRIORITY_MAX,
1505                                    smoke->batch);
1506                 mutex_unlock(&smoke->i915->drm.struct_mutex);
1507                 if (err)
1508                         return err;
1509
1510                 count++;
1511         } while (!__igt_timeout(end_time, NULL));
1512
1513         smoke->count = count;
1514         return 0;
1515 }
1516
1517 static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
1518 #define BATCH BIT(0)
1519 {
1520         struct task_struct *tsk[I915_NUM_ENGINES] = {};
1521         struct preempt_smoke arg[I915_NUM_ENGINES];
1522         struct intel_engine_cs *engine;
1523         enum intel_engine_id id;
1524         unsigned long count;
1525         int err = 0;
1526
1527         mutex_unlock(&smoke->i915->drm.struct_mutex);
1528
1529         for_each_engine(engine, smoke->i915, id) {
1530                 arg[id] = *smoke;
1531                 arg[id].engine = engine;
1532                 if (!(flags & BATCH))
1533                         arg[id].batch = NULL;
1534                 arg[id].count = 0;
1535
1536                 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
1537                                       "igt/smoke:%d", id);
1538                 if (IS_ERR(tsk[id])) {
1539                         err = PTR_ERR(tsk[id]);
1540                         break;
1541                 }
1542                 get_task_struct(tsk[id]);
1543         }
1544
1545         count = 0;
1546         for_each_engine(engine, smoke->i915, id) {
1547                 int status;
1548
1549                 if (IS_ERR_OR_NULL(tsk[id]))
1550                         continue;
1551
1552                 status = kthread_stop(tsk[id]);
1553                 if (status && !err)
1554                         err = status;
1555
1556                 count += arg[id].count;
1557
1558                 put_task_struct(tsk[id]);
1559         }
1560
1561         mutex_lock(&smoke->i915->drm.struct_mutex);
1562
1563         pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
1564                 count, flags,
1565                 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext);
1566         return 0;
1567 }
1568
1569 static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
1570 {
1571         enum intel_engine_id id;
1572         IGT_TIMEOUT(end_time);
1573         unsigned long count;
1574
1575         count = 0;
1576         do {
1577                 for_each_engine(smoke->engine, smoke->i915, id) {
1578                         struct i915_gem_context *ctx = smoke_context(smoke);
1579                         int err;
1580
1581                         err = smoke_submit(smoke,
1582                                            ctx, random_priority(&smoke->prng),
1583                                            flags & BATCH ? smoke->batch : NULL);
1584                         if (err)
1585                                 return err;
1586
1587                         count++;
1588                 }
1589         } while (!__igt_timeout(end_time, NULL));
1590
1591         pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
1592                 count, flags,
1593                 RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext);
1594         return 0;
1595 }
1596
1597 static int live_preempt_smoke(void *arg)
1598 {
1599         struct preempt_smoke smoke = {
1600                 .i915 = arg,
1601                 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
1602                 .ncontext = 1024,
1603         };
1604         const unsigned int phase[] = { 0, BATCH };
1605         intel_wakeref_t wakeref;
1606         struct igt_live_test t;
1607         int err = -ENOMEM;
1608         u32 *cs;
1609         int n;
1610
1611         if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915))
1612                 return 0;
1613
1614         smoke.contexts = kmalloc_array(smoke.ncontext,
1615                                        sizeof(*smoke.contexts),
1616                                        GFP_KERNEL);
1617         if (!smoke.contexts)
1618                 return -ENOMEM;
1619
1620         mutex_lock(&smoke.i915->drm.struct_mutex);
1621         wakeref = intel_runtime_pm_get(&smoke.i915->runtime_pm);
1622
1623         smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE);
1624         if (IS_ERR(smoke.batch)) {
1625                 err = PTR_ERR(smoke.batch);
1626                 goto err_unlock;
1627         }
1628
1629         cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
1630         if (IS_ERR(cs)) {
1631                 err = PTR_ERR(cs);
1632                 goto err_batch;
1633         }
1634         for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
1635                 cs[n] = MI_ARB_CHECK;
1636         cs[n] = MI_BATCH_BUFFER_END;
1637         i915_gem_object_flush_map(smoke.batch);
1638         i915_gem_object_unpin_map(smoke.batch);
1639
1640         if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) {
1641                 err = -EIO;
1642                 goto err_batch;
1643         }
1644
1645         for (n = 0; n < smoke.ncontext; n++) {
1646                 smoke.contexts[n] = kernel_context(smoke.i915);
1647                 if (!smoke.contexts[n])
1648                         goto err_ctx;
1649         }
1650
1651         for (n = 0; n < ARRAY_SIZE(phase); n++) {
1652                 err = smoke_crescendo(&smoke, phase[n]);
1653                 if (err)
1654                         goto err_ctx;
1655
1656                 err = smoke_random(&smoke, phase[n]);
1657                 if (err)
1658                         goto err_ctx;
1659         }
1660
1661 err_ctx:
1662         if (igt_live_test_end(&t))
1663                 err = -EIO;
1664
1665         for (n = 0; n < smoke.ncontext; n++) {
1666                 if (!smoke.contexts[n])
1667                         break;
1668                 kernel_context_close(smoke.contexts[n]);
1669         }
1670
1671 err_batch:
1672         i915_gem_object_put(smoke.batch);
1673 err_unlock:
1674         intel_runtime_pm_put(&smoke.i915->runtime_pm, wakeref);
1675         mutex_unlock(&smoke.i915->drm.struct_mutex);
1676         kfree(smoke.contexts);
1677
1678         return err;
1679 }
1680
1681 static int nop_virtual_engine(struct drm_i915_private *i915,
1682                               struct intel_engine_cs **siblings,
1683                               unsigned int nsibling,
1684                               unsigned int nctx,
1685                               unsigned int flags)
1686 #define CHAIN BIT(0)
1687 {
1688         IGT_TIMEOUT(end_time);
1689         struct i915_request *request[16];
1690         struct i915_gem_context *ctx[16];
1691         struct intel_context *ve[16];
1692         unsigned long n, prime, nc;
1693         struct igt_live_test t;
1694         ktime_t times[2] = {};
1695         int err;
1696
1697         GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx));
1698
1699         for (n = 0; n < nctx; n++) {
1700                 ctx[n] = kernel_context(i915);
1701                 if (!ctx[n]) {
1702                         err = -ENOMEM;
1703                         nctx = n;
1704                         goto out;
1705                 }
1706
1707                 ve[n] = intel_execlists_create_virtual(ctx[n],
1708                                                        siblings, nsibling);
1709                 if (IS_ERR(ve[n])) {
1710                         kernel_context_close(ctx[n]);
1711                         err = PTR_ERR(ve[n]);
1712                         nctx = n;
1713                         goto out;
1714                 }
1715
1716                 err = intel_context_pin(ve[n]);
1717                 if (err) {
1718                         intel_context_put(ve[n]);
1719                         kernel_context_close(ctx[n]);
1720                         nctx = n;
1721                         goto out;
1722                 }
1723         }
1724
1725         err = igt_live_test_begin(&t, i915, __func__, ve[0]->engine->name);
1726         if (err)
1727                 goto out;
1728
1729         for_each_prime_number_from(prime, 1, 8192) {
1730                 times[1] = ktime_get_raw();
1731
1732                 if (flags & CHAIN) {
1733                         for (nc = 0; nc < nctx; nc++) {
1734                                 for (n = 0; n < prime; n++) {
1735                                         request[nc] =
1736                                                 i915_request_create(ve[nc]);
1737                                         if (IS_ERR(request[nc])) {
1738                                                 err = PTR_ERR(request[nc]);
1739                                                 goto out;
1740                                         }
1741
1742                                         i915_request_add(request[nc]);
1743                                 }
1744                         }
1745                 } else {
1746                         for (n = 0; n < prime; n++) {
1747                                 for (nc = 0; nc < nctx; nc++) {
1748                                         request[nc] =
1749                                                 i915_request_create(ve[nc]);
1750                                         if (IS_ERR(request[nc])) {
1751                                                 err = PTR_ERR(request[nc]);
1752                                                 goto out;
1753                                         }
1754
1755                                         i915_request_add(request[nc]);
1756                                 }
1757                         }
1758                 }
1759
1760                 for (nc = 0; nc < nctx; nc++) {
1761                         if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
1762                                 pr_err("%s(%s): wait for %llx:%lld timed out\n",
1763                                        __func__, ve[0]->engine->name,
1764                                        request[nc]->fence.context,
1765                                        request[nc]->fence.seqno);
1766
1767                                 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
1768                                           __func__, ve[0]->engine->name,
1769                                           request[nc]->fence.context,
1770                                           request[nc]->fence.seqno);
1771                                 GEM_TRACE_DUMP();
1772                                 intel_gt_set_wedged(&i915->gt);
1773                                 break;
1774                         }
1775                 }
1776
1777                 times[1] = ktime_sub(ktime_get_raw(), times[1]);
1778                 if (prime == 1)
1779                         times[0] = times[1];
1780
1781                 if (__igt_timeout(end_time, NULL))
1782                         break;
1783         }
1784
1785         err = igt_live_test_end(&t);
1786         if (err)
1787                 goto out;
1788
1789         pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
1790                 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
1791                 prime, div64_u64(ktime_to_ns(times[1]), prime));
1792
1793 out:
1794         if (igt_flush_test(i915, I915_WAIT_LOCKED))
1795                 err = -EIO;
1796
1797         for (nc = 0; nc < nctx; nc++) {
1798                 intel_context_unpin(ve[nc]);
1799                 intel_context_put(ve[nc]);
1800                 kernel_context_close(ctx[nc]);
1801         }
1802         return err;
1803 }
1804
1805 static int live_virtual_engine(void *arg)
1806 {
1807         struct drm_i915_private *i915 = arg;
1808         struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
1809         struct intel_engine_cs *engine;
1810         struct intel_gt *gt = &i915->gt;
1811         enum intel_engine_id id;
1812         unsigned int class, inst;
1813         int err = -ENODEV;
1814
1815         if (USES_GUC_SUBMISSION(i915))
1816                 return 0;
1817
1818         mutex_lock(&i915->drm.struct_mutex);
1819
1820         for_each_engine(engine, i915, id) {
1821                 err = nop_virtual_engine(i915, &engine, 1, 1, 0);
1822                 if (err) {
1823                         pr_err("Failed to wrap engine %s: err=%d\n",
1824                                engine->name, err);
1825                         goto out_unlock;
1826                 }
1827         }
1828
1829         for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
1830                 int nsibling, n;
1831
1832                 nsibling = 0;
1833                 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
1834                         if (!gt->engine_class[class][inst])
1835                                 continue;
1836
1837                         siblings[nsibling++] = gt->engine_class[class][inst];
1838                 }
1839                 if (nsibling < 2)
1840                         continue;
1841
1842                 for (n = 1; n <= nsibling + 1; n++) {
1843                         err = nop_virtual_engine(i915, siblings, nsibling,
1844                                                  n, 0);
1845                         if (err)
1846                                 goto out_unlock;
1847                 }
1848
1849                 err = nop_virtual_engine(i915, siblings, nsibling, n, CHAIN);
1850                 if (err)
1851                         goto out_unlock;
1852         }
1853
1854 out_unlock:
1855         mutex_unlock(&i915->drm.struct_mutex);
1856         return err;
1857 }
1858
1859 static int mask_virtual_engine(struct drm_i915_private *i915,
1860                                struct intel_engine_cs **siblings,
1861                                unsigned int nsibling)
1862 {
1863         struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
1864         struct i915_gem_context *ctx;
1865         struct intel_context *ve;
1866         struct igt_live_test t;
1867         unsigned int n;
1868         int err;
1869
1870         /*
1871          * Check that by setting the execution mask on a request, we can
1872          * restrict it to our desired engine within the virtual engine.
1873          */
1874
1875         ctx = kernel_context(i915);
1876         if (!ctx)
1877                 return -ENOMEM;
1878
1879         ve = intel_execlists_create_virtual(ctx, siblings, nsibling);
1880         if (IS_ERR(ve)) {
1881                 err = PTR_ERR(ve);
1882                 goto out_close;
1883         }
1884
1885         err = intel_context_pin(ve);
1886         if (err)
1887                 goto out_put;
1888
1889         err = igt_live_test_begin(&t, i915, __func__, ve->engine->name);
1890         if (err)
1891                 goto out_unpin;
1892
1893         for (n = 0; n < nsibling; n++) {
1894                 request[n] = i915_request_create(ve);
1895                 if (IS_ERR(request[n])) {
1896                         err = PTR_ERR(request[n]);
1897                         nsibling = n;
1898                         goto out;
1899                 }
1900
1901                 /* Reverse order as it's more likely to be unnatural */
1902                 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
1903
1904                 i915_request_get(request[n]);
1905                 i915_request_add(request[n]);
1906         }
1907
1908         for (n = 0; n < nsibling; n++) {
1909                 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
1910                         pr_err("%s(%s): wait for %llx:%lld timed out\n",
1911                                __func__, ve->engine->name,
1912                                request[n]->fence.context,
1913                                request[n]->fence.seqno);
1914
1915                         GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
1916                                   __func__, ve->engine->name,
1917                                   request[n]->fence.context,
1918                                   request[n]->fence.seqno);
1919                         GEM_TRACE_DUMP();
1920                         intel_gt_set_wedged(&i915->gt);
1921                         err = -EIO;
1922                         goto out;
1923                 }
1924
1925                 if (request[n]->engine != siblings[nsibling - n - 1]) {
1926                         pr_err("Executed on wrong sibling '%s', expected '%s'\n",
1927                                request[n]->engine->name,
1928                                siblings[nsibling - n - 1]->name);
1929                         err = -EINVAL;
1930                         goto out;
1931                 }
1932         }
1933
1934         err = igt_live_test_end(&t);
1935         if (err)
1936                 goto out;
1937
1938 out:
1939         if (igt_flush_test(i915, I915_WAIT_LOCKED))
1940                 err = -EIO;
1941
1942         for (n = 0; n < nsibling; n++)
1943                 i915_request_put(request[n]);
1944
1945 out_unpin:
1946         intel_context_unpin(ve);
1947 out_put:
1948         intel_context_put(ve);
1949 out_close:
1950         kernel_context_close(ctx);
1951         return err;
1952 }
1953
1954 static int live_virtual_mask(void *arg)
1955 {
1956         struct drm_i915_private *i915 = arg;
1957         struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
1958         struct intel_gt *gt = &i915->gt;
1959         unsigned int class, inst;
1960         int err = 0;
1961
1962         if (USES_GUC_SUBMISSION(i915))
1963                 return 0;
1964
1965         mutex_lock(&i915->drm.struct_mutex);
1966
1967         for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
1968                 unsigned int nsibling;
1969
1970                 nsibling = 0;
1971                 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
1972                         if (!gt->engine_class[class][inst])
1973                                 break;
1974
1975                         siblings[nsibling++] = gt->engine_class[class][inst];
1976                 }
1977                 if (nsibling < 2)
1978                         continue;
1979
1980                 err = mask_virtual_engine(i915, siblings, nsibling);
1981                 if (err)
1982                         goto out_unlock;
1983         }
1984
1985 out_unlock:
1986         mutex_unlock(&i915->drm.struct_mutex);
1987         return err;
1988 }
1989
1990 static int bond_virtual_engine(struct drm_i915_private *i915,
1991                                unsigned int class,
1992                                struct intel_engine_cs **siblings,
1993                                unsigned int nsibling,
1994                                unsigned int flags)
1995 #define BOND_SCHEDULE BIT(0)
1996 {
1997         struct intel_engine_cs *master;
1998         struct i915_gem_context *ctx;
1999         struct i915_request *rq[16];
2000         enum intel_engine_id id;
2001         unsigned long n;
2002         int err;
2003
2004         GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
2005
2006         ctx = kernel_context(i915);
2007         if (!ctx)
2008                 return -ENOMEM;
2009
2010         err = 0;
2011         rq[0] = ERR_PTR(-ENOMEM);
2012         for_each_engine(master, i915, id) {
2013                 struct i915_sw_fence fence = {};
2014
2015                 if (master->class == class)
2016                         continue;
2017
2018                 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
2019
2020                 rq[0] = igt_request_alloc(ctx, master);
2021                 if (IS_ERR(rq[0])) {
2022                         err = PTR_ERR(rq[0]);
2023                         goto out;
2024                 }
2025                 i915_request_get(rq[0]);
2026
2027                 if (flags & BOND_SCHEDULE) {
2028                         onstack_fence_init(&fence);
2029                         err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
2030                                                                &fence,
2031                                                                GFP_KERNEL);
2032                 }
2033                 i915_request_add(rq[0]);
2034                 if (err < 0)
2035                         goto out;
2036
2037                 for (n = 0; n < nsibling; n++) {
2038                         struct intel_context *ve;
2039
2040                         ve = intel_execlists_create_virtual(ctx,
2041                                                             siblings,
2042                                                             nsibling);
2043                         if (IS_ERR(ve)) {
2044                                 err = PTR_ERR(ve);
2045                                 onstack_fence_fini(&fence);
2046                                 goto out;
2047                         }
2048
2049                         err = intel_virtual_engine_attach_bond(ve->engine,
2050                                                                master,
2051                                                                siblings[n]);
2052                         if (err) {
2053                                 intel_context_put(ve);
2054                                 onstack_fence_fini(&fence);
2055                                 goto out;
2056                         }
2057
2058                         err = intel_context_pin(ve);
2059                         intel_context_put(ve);
2060                         if (err) {
2061                                 onstack_fence_fini(&fence);
2062                                 goto out;
2063                         }
2064
2065                         rq[n + 1] = i915_request_create(ve);
2066                         intel_context_unpin(ve);
2067                         if (IS_ERR(rq[n + 1])) {
2068                                 err = PTR_ERR(rq[n + 1]);
2069                                 onstack_fence_fini(&fence);
2070                                 goto out;
2071                         }
2072                         i915_request_get(rq[n + 1]);
2073
2074                         err = i915_request_await_execution(rq[n + 1],
2075                                                            &rq[0]->fence,
2076                                                            ve->engine->bond_execute);
2077                         i915_request_add(rq[n + 1]);
2078                         if (err < 0) {
2079                                 onstack_fence_fini(&fence);
2080                                 goto out;
2081                         }
2082                 }
2083                 onstack_fence_fini(&fence);
2084
2085                 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
2086                         pr_err("Master request did not execute (on %s)!\n",
2087                                rq[0]->engine->name);
2088                         err = -EIO;
2089                         goto out;
2090                 }
2091
2092                 for (n = 0; n < nsibling; n++) {
2093                         if (i915_request_wait(rq[n + 1], 0,
2094                                               MAX_SCHEDULE_TIMEOUT) < 0) {
2095                                 err = -EIO;
2096                                 goto out;
2097                         }
2098
2099                         if (rq[n + 1]->engine != siblings[n]) {
2100                                 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
2101                                        siblings[n]->name,
2102                                        rq[n + 1]->engine->name,
2103                                        rq[0]->engine->name);
2104                                 err = -EINVAL;
2105                                 goto out;
2106                         }
2107                 }
2108
2109                 for (n = 0; !IS_ERR(rq[n]); n++)
2110                         i915_request_put(rq[n]);
2111                 rq[0] = ERR_PTR(-ENOMEM);
2112         }
2113
2114 out:
2115         for (n = 0; !IS_ERR(rq[n]); n++)
2116                 i915_request_put(rq[n]);
2117         if (igt_flush_test(i915, I915_WAIT_LOCKED))
2118                 err = -EIO;
2119
2120         kernel_context_close(ctx);
2121         return err;
2122 }
2123
2124 static int live_virtual_bond(void *arg)
2125 {
2126         static const struct phase {
2127                 const char *name;
2128                 unsigned int flags;
2129         } phases[] = {
2130                 { "", 0 },
2131                 { "schedule", BOND_SCHEDULE },
2132                 { },
2133         };
2134         struct drm_i915_private *i915 = arg;
2135         struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
2136         struct intel_gt *gt = &i915->gt;
2137         unsigned int class, inst;
2138         int err = 0;
2139
2140         if (USES_GUC_SUBMISSION(i915))
2141                 return 0;
2142
2143         mutex_lock(&i915->drm.struct_mutex);
2144
2145         for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
2146                 const struct phase *p;
2147                 int nsibling;
2148
2149                 nsibling = 0;
2150                 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
2151                         if (!gt->engine_class[class][inst])
2152                                 break;
2153
2154                         GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings));
2155                         siblings[nsibling++] = gt->engine_class[class][inst];
2156                 }
2157                 if (nsibling < 2)
2158                         continue;
2159
2160                 for (p = phases; p->name; p++) {
2161                         err = bond_virtual_engine(i915,
2162                                                   class, siblings, nsibling,
2163                                                   p->flags);
2164                         if (err) {
2165                                 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
2166                                        __func__, p->name, class, nsibling, err);
2167                                 goto out_unlock;
2168                         }
2169                 }
2170         }
2171
2172 out_unlock:
2173         mutex_unlock(&i915->drm.struct_mutex);
2174         return err;
2175 }
2176
2177 int intel_execlists_live_selftests(struct drm_i915_private *i915)
2178 {
2179         static const struct i915_subtest tests[] = {
2180                 SUBTEST(live_sanitycheck),
2181                 SUBTEST(live_timeslice_preempt),
2182                 SUBTEST(live_busywait_preempt),
2183                 SUBTEST(live_preempt),
2184                 SUBTEST(live_late_preempt),
2185                 SUBTEST(live_nopreempt),
2186                 SUBTEST(live_suppress_self_preempt),
2187                 SUBTEST(live_suppress_wait_preempt),
2188                 SUBTEST(live_chain_preempt),
2189                 SUBTEST(live_preempt_hang),
2190                 SUBTEST(live_preempt_smoke),
2191                 SUBTEST(live_virtual_engine),
2192                 SUBTEST(live_virtual_mask),
2193                 SUBTEST(live_virtual_bond),
2194         };
2195
2196         if (!HAS_EXECLISTS(i915))
2197                 return 0;
2198
2199         if (intel_gt_is_wedged(&i915->gt))
2200                 return 0;
2201
2202         return i915_live_subtests(tests, i915);
2203 }
2204
2205 static void hexdump(const void *buf, size_t len)
2206 {
2207         const size_t rowsize = 8 * sizeof(u32);
2208         const void *prev = NULL;
2209         bool skip = false;
2210         size_t pos;
2211
2212         for (pos = 0; pos < len; pos += rowsize) {
2213                 char line[128];
2214
2215                 if (prev && !memcmp(prev, buf + pos, rowsize)) {
2216                         if (!skip) {
2217                                 pr_info("*\n");
2218                                 skip = true;
2219                         }
2220                         continue;
2221                 }
2222
2223                 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
2224                                                 rowsize, sizeof(u32),
2225                                                 line, sizeof(line),
2226                                                 false) >= sizeof(line));
2227                 pr_info("[%04zx] %s\n", pos, line);
2228
2229                 prev = buf + pos;
2230                 skip = false;
2231         }
2232 }
2233
2234 static int live_lrc_layout(void *arg)
2235 {
2236         struct intel_gt *gt = arg;
2237         struct intel_engine_cs *engine;
2238         enum intel_engine_id id;
2239         u32 *mem;
2240         int err;
2241
2242         /*
2243          * Check the registers offsets we use to create the initial reg state
2244          * match the layout saved by HW.
2245          */
2246
2247         mem = kmalloc(PAGE_SIZE, GFP_KERNEL);
2248         if (!mem)
2249                 return -ENOMEM;
2250
2251         err = 0;
2252         for_each_engine(engine, gt->i915, id) {
2253                 u32 *hw, *lrc;
2254                 int dw;
2255
2256                 if (!engine->default_state)
2257                         continue;
2258
2259                 hw = i915_gem_object_pin_map(engine->default_state,
2260                                              I915_MAP_WB);
2261                 if (IS_ERR(hw)) {
2262                         err = PTR_ERR(hw);
2263                         break;
2264                 }
2265                 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
2266
2267                 lrc = memset(mem, 0, PAGE_SIZE);
2268                 execlists_init_reg_state(lrc,
2269                                          engine->kernel_context,
2270                                          engine,
2271                                          engine->kernel_context->ring,
2272                                          true);
2273
2274                 dw = 0;
2275                 do {
2276                         u32 lri = hw[dw];
2277
2278                         if (lri == 0) {
2279                                 dw++;
2280                                 continue;
2281                         }
2282
2283                         if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
2284                                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
2285                                        engine->name, dw, lri);
2286                                 err = -EINVAL;
2287                                 break;
2288                         }
2289
2290                         if (lrc[dw] != lri) {
2291                                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
2292                                        engine->name, dw, lri, lrc[dw]);
2293                                 err = -EINVAL;
2294                                 break;
2295                         }
2296
2297                         lri &= 0x7f;
2298                         lri++;
2299                         dw++;
2300
2301                         while (lri) {
2302                                 if (hw[dw] != lrc[dw]) {
2303                                         pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
2304                                                engine->name, dw, hw[dw], lrc[dw]);
2305                                         err = -EINVAL;
2306                                         break;
2307                                 }
2308
2309                                 /*
2310                                  * Skip over the actual register value as we
2311                                  * expect that to differ.
2312                                  */
2313                                 dw += 2;
2314                                 lri -= 2;
2315                         }
2316                 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
2317
2318                 if (err) {
2319                         pr_info("%s: HW register image:\n", engine->name);
2320                         hexdump(hw, PAGE_SIZE);
2321
2322                         pr_info("%s: SW register image:\n", engine->name);
2323                         hexdump(lrc, PAGE_SIZE);
2324                 }
2325
2326                 i915_gem_object_unpin_map(engine->default_state);
2327                 if (err)
2328                         break;
2329         }
2330
2331         kfree(mem);
2332         return err;
2333 }
2334
2335 int intel_lrc_live_selftests(struct drm_i915_private *i915)
2336 {
2337         static const struct i915_subtest tests[] = {
2338                 SUBTEST(live_lrc_layout),
2339         };
2340
2341         if (!HAS_LOGICAL_RING_CONTEXTS(i915))
2342                 return 0;
2343
2344         return intel_gt_live_subtests(tests, &i915->gt);
2345 }