drivers/gpu/drm/i915/intel_engine_cs.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_print.h>
  26
  27 #include "i915_drv.h"
  28 #include "i915_reset.h"
  29 #include "intel_ringbuffer.h"
  30 #include "intel_lrc.h"
  31
  32 /* Haswell does have the CXT_SIZE register however it does not appear to be
  33  * valid. Now, docs explain in dwords what is in the context object. The full
  34  * size is 70720 bytes, however, the power context and execlist context will
  35  * never be saved (power context is stored elsewhere, and execlists don't work
  36  * on HSW) - so the final size, including the extra state required for the
  37  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  38  */
  39 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
  40
  41 #define DEFAULT_LR_CONTEXT_RENDER_SIZE  (22 * PAGE_SIZE)
  42 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
  43 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
  44 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
  45 #define GEN11_LR_CONTEXT_RENDER_SIZE    (14 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
  48
  49 struct engine_class_info {
  50         const char *name;
  51         int (*init_legacy)(struct intel_engine_cs *engine);
  52         int (*init_execlists)(struct intel_engine_cs *engine);
  53
  54         u8 uabi_class;
  55 };
  56
  57 static const struct engine_class_info intel_engine_classes[] = {
  58         [RENDER_CLASS] = {
  59                 .name = "rcs",
  60                 .init_execlists = logical_render_ring_init,
  61                 .init_legacy = intel_init_render_ring_buffer,
  62                 .uabi_class = I915_ENGINE_CLASS_RENDER,
  63         },
  64         [COPY_ENGINE_CLASS] = {
  65                 .name = "bcs",
  66                 .init_execlists = logical_xcs_ring_init,
  67                 .init_legacy = intel_init_blt_ring_buffer,
  68                 .uabi_class = I915_ENGINE_CLASS_COPY,
  69         },
  70         [VIDEO_DECODE_CLASS] = {
  71                 .name = "vcs",
  72                 .init_execlists = logical_xcs_ring_init,
  73                 .init_legacy = intel_init_bsd_ring_buffer,
  74                 .uabi_class = I915_ENGINE_CLASS_VIDEO,
  75         },
  76         [VIDEO_ENHANCEMENT_CLASS] = {
  77                 .name = "vecs",
  78                 .init_execlists = logical_xcs_ring_init,
  79                 .init_legacy = intel_init_vebox_ring_buffer,
  80                 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
  81         },
  82 };
  83
  84 #define MAX_MMIO_BASES 3
  85 struct engine_info {
  86         unsigned int hw_id;
  87         u8 class;
  88         u8 instance;
  89         /* mmio bases table *must* be sorted in reverse gen order */
  90         struct engine_mmio_base {
  91                 u32 gen : 8;
  92                 u32 base : 24;
  93         } mmio_bases[MAX_MMIO_BASES];
  94 };
  95
  96 static const struct engine_info intel_engines[] = {
  97         [RCS0] = {
  98                 .hw_id = RCS0_HW,
  99                 .class = RENDER_CLASS,
 100                 .instance = 0,
 101                 .mmio_bases = {
 102                         { .gen = 1, .base = RENDER_RING_BASE }
 103                 },
 104         },
 105         [BCS0] = {
 106                 .hw_id = BCS0_HW,
 107                 .class = COPY_ENGINE_CLASS,
 108                 .instance = 0,
 109                 .mmio_bases = {
 110                         { .gen = 6, .base = BLT_RING_BASE }
 111                 },
 112         },
 113         [VCS0] = {
 114                 .hw_id = VCS0_HW,
 115                 .class = VIDEO_DECODE_CLASS,
 116                 .instance = 0,
 117                 .mmio_bases = {
 118                         { .gen = 11, .base = GEN11_BSD_RING_BASE },
 119                         { .gen = 6, .base = GEN6_BSD_RING_BASE },
 120                         { .gen = 4, .base = BSD_RING_BASE }
 121                 },
 122         },
 123         [VCS1] = {
 124                 .hw_id = VCS1_HW,
 125                 .class = VIDEO_DECODE_CLASS,
 126                 .instance = 1,
 127                 .mmio_bases = {
 128                         { .gen = 11, .base = GEN11_BSD2_RING_BASE },
 129                         { .gen = 8, .base = GEN8_BSD2_RING_BASE }
 130                 },
 131         },
 132         [VCS2] = {
 133                 .hw_id = VCS2_HW,
 134                 .class = VIDEO_DECODE_CLASS,
 135                 .instance = 2,
 136                 .mmio_bases = {
 137                         { .gen = 11, .base = GEN11_BSD3_RING_BASE }
 138                 },
 139         },
 140         [VCS3] = {
 141                 .hw_id = VCS3_HW,
 142                 .class = VIDEO_DECODE_CLASS,
 143                 .instance = 3,
 144                 .mmio_bases = {
 145                         { .gen = 11, .base = GEN11_BSD4_RING_BASE }
 146                 },
 147         },
 148         [VECS0] = {
 149                 .hw_id = VECS0_HW,
 150                 .class = VIDEO_ENHANCEMENT_CLASS,
 151                 .instance = 0,
 152                 .mmio_bases = {
 153                         { .gen = 11, .base = GEN11_VEBOX_RING_BASE },
 154                         { .gen = 7, .base = VEBOX_RING_BASE }
 155                 },
 156         },
 157         [VECS1] = {
 158                 .hw_id = VECS1_HW,
 159                 .class = VIDEO_ENHANCEMENT_CLASS,
 160                 .instance = 1,
 161                 .mmio_bases = {
 162                         { .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
 163                 },
 164         },
 165 };
 166
 167 /**
 168  * ___intel_engine_context_size() - return the size of the context for an engine
 169  * @dev_priv: i915 device private
 170  * @class: engine class
 171  *
 172  * Each engine class may require a different amount of space for a context
 173  * image.
 174  *
 175  * Return: size (in bytes) of an engine class specific context image
 176  *
 177  * Note: this size includes the HWSP, which is part of the context image
 178  * in LRC mode, but does not include the "shared data page" used with
 179  * GuC submission. The caller should account for this if using the GuC.
 180  */
 181 static u32
 182 __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 183 {
 184         u32 cxt_size;
 185
 186         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
 187
 188         switch (class) {
 189         case RENDER_CLASS:
 190                 switch (INTEL_GEN(dev_priv)) {
 191                 default:
 192                         MISSING_CASE(INTEL_GEN(dev_priv));
 193                         return DEFAULT_LR_CONTEXT_RENDER_SIZE;
 194                 case 11:
 195                         return GEN11_LR_CONTEXT_RENDER_SIZE;
 196                 case 10:
 197                         return GEN10_LR_CONTEXT_RENDER_SIZE;
 198                 case 9:
 199                         return GEN9_LR_CONTEXT_RENDER_SIZE;
 200                 case 8:
 201                         return GEN8_LR_CONTEXT_RENDER_SIZE;
 202                 case 7:
 203                         if (IS_HASWELL(dev_priv))
 204                                 return HSW_CXT_TOTAL_SIZE;
 205
 206                         cxt_size = I915_READ(GEN7_CXT_SIZE);
 207                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
 208                                         PAGE_SIZE);
 209                 case 6:
 210                         cxt_size = I915_READ(CXT_SIZE);
 211                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
 212                                         PAGE_SIZE);
 213                 case 5:
 214                 case 4:
 215                 case 3:
 216                 case 2:
 217                 /* For the special day when i810 gets merged. */
 218                 case 1:
 219                         return 0;
 220                 }
 221                 break;
 222         default:
 223                 MISSING_CASE(class);
 224                 /* fall through */
 225         case VIDEO_DECODE_CLASS:
 226         case VIDEO_ENHANCEMENT_CLASS:
 227         case COPY_ENGINE_CLASS:
 228                 if (INTEL_GEN(dev_priv) < 8)
 229                         return 0;
 230                 return GEN8_LR_CONTEXT_OTHER_SIZE;
 231         }
 232 }
 233
 234 static u32 __engine_mmio_base(struct drm_i915_private *i915,
 235                               const struct engine_mmio_base *bases)
 236 {
 237         int i;
 238
 239         for (i = 0; i < MAX_MMIO_BASES; i++)
 240                 if (INTEL_GEN(i915) >= bases[i].gen)
 241                         break;
 242
 243         GEM_BUG_ON(i == MAX_MMIO_BASES);
 244         GEM_BUG_ON(!bases[i].base);
 245
 246         return bases[i].base;
 247 }
 248
 249 static void __sprint_engine_name(char *name, const struct engine_info *info)
 250 {
 251         WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
 252                          intel_engine_classes[info->class].name,
 253                          info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
 254 }
 255
 256 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
 257 {
 258         struct drm_i915_private *dev_priv = engine->i915;
 259         i915_reg_t hwstam;
 260
 261         /*
 262          * Though they added more rings on g4x/ilk, they did not add
 263          * per-engine HWSTAM until gen6.
 264          */
 265         if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS)
 266                 return;
 267
 268         hwstam = RING_HWSTAM(engine->mmio_base);
 269         if (INTEL_GEN(dev_priv) >= 3)
 270                 I915_WRITE(hwstam, mask);
 271         else
 272                 I915_WRITE16(hwstam, mask);
 273 }
 274
 275 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
 276 {
 277         /* Mask off all writes into the unknown HWSP */
 278         intel_engine_set_hwsp_writemask(engine, ~0u);
 279 }
 280
 281 static int
 282 intel_engine_setup(struct drm_i915_private *dev_priv,
 283                    enum intel_engine_id id)
 284 {
 285         const struct engine_info *info = &intel_engines[id];
 286         struct intel_engine_cs *engine;
 287
 288         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 289
 290         BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
 291         BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
 292
 293         if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
 294                 return -EINVAL;
 295
 296         if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
 297                 return -EINVAL;
 298
 299         if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
 300                 return -EINVAL;
 301
 302         GEM_BUG_ON(dev_priv->engine[id]);
 303         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 304         if (!engine)
 305                 return -ENOMEM;
 306
 307         BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
 308
 309         engine->id = id;
 310         engine->mask = BIT(id);
 311         engine->i915 = dev_priv;
 312         __sprint_engine_name(engine->name, info);
 313         engine->hw_id = engine->guc_id = info->hw_id;
 314         engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 315         engine->class = info->class;
 316         engine->instance = info->instance;
 317
 318         engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 319
 320         engine->context_size = __intel_engine_context_size(dev_priv,
 321                                                            engine->class);
 322         if (WARN_ON(engine->context_size > BIT(20)))
 323                 engine->context_size = 0;
 324         if (engine->context_size)
 325                 DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 326
 327         /* Nothing to do here, execute in order of dependencies */
 328         engine->schedule = NULL;
 329
 330         seqlock_init(&engine->stats.lock);
 331
 332         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 333
 334         /* Scrub mmio state on takeover */
 335         intel_engine_sanitize_mmio(engine);
 336
 337         dev_priv->engine_class[info->class][info->instance] = engine;
 338         dev_priv->engine[id] = engine;
 339         return 0;
 340 }
 341
 342 /**
 343  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
 344  * @dev_priv: i915 device private
 345  *
 346  * Return: non-zero if the initialization failed.
 347  */
 348 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 349 {
 350         struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
 351         const unsigned int engine_mask = INTEL_INFO(dev_priv)->engine_mask;
 352         struct intel_engine_cs *engine;
 353         enum intel_engine_id id;
 354         unsigned int mask = 0;
 355         unsigned int i;
 356         int err;
 357
 358         WARN_ON(engine_mask == 0);
 359         WARN_ON(engine_mask &
 360                 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 361
 362         if (i915_inject_load_failure())
 363                 return -ENODEV;
 364
 365         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
 366                 if (!HAS_ENGINE(dev_priv, i))
 367                         continue;
 368
 369                 err = intel_engine_setup(dev_priv, i);
 370                 if (err)
 371                         goto cleanup;
 372
 373                 mask |= BIT(i);
 374         }
 375
 376         /*
 377          * Catch failures to update intel_engines table when the new engines
 378          * are added to the driver by a warning and disabling the forgotten
 379          * engines.
 380          */
 381         if (WARN_ON(mask != engine_mask))
 382                 device_info->engine_mask = mask;
 383
 384         /* We always presume we have at least RCS available for later probing */
 385         if (WARN_ON(!HAS_ENGINE(dev_priv, RCS0))) {
 386                 err = -ENODEV;
 387                 goto cleanup;
 388         }
 389
 390         RUNTIME_INFO(dev_priv)->num_engines = hweight32(mask);
 391
 392         i915_check_and_clear_faults(dev_priv);
 393
 394         return 0;
 395
 396 cleanup:
 397         for_each_engine(engine, dev_priv, id)
 398                 kfree(engine);
 399         return err;
 400 }
 401
 402 /**
 403  * intel_engines_init() - init the Engine Command Streamers
 404  * @dev_priv: i915 device private
 405  *
 406  * Return: non-zero if the initialization failed.
 407  */
 408 int intel_engines_init(struct drm_i915_private *dev_priv)
 409 {
 410         struct intel_engine_cs *engine;
 411         enum intel_engine_id id, err_id;
 412         int err;
 413
 414         for_each_engine(engine, dev_priv, id) {
 415                 const struct engine_class_info *class_info =
 416                         &intel_engine_classes[engine->class];
 417                 int (*init)(struct intel_engine_cs *engine);
 418
 419                 if (HAS_EXECLISTS(dev_priv))
 420                         init = class_info->init_execlists;
 421                 else
 422                         init = class_info->init_legacy;
 423
 424                 err = -EINVAL;
 425                 err_id = id;
 426
 427                 if (GEM_DEBUG_WARN_ON(!init))
 428                         goto cleanup;
 429
 430                 err = init(engine);
 431                 if (err)
 432                         goto cleanup;
 433
 434                 GEM_BUG_ON(!engine->submit_request);
 435         }
 436
 437         return 0;
 438
 439 cleanup:
 440         for_each_engine(engine, dev_priv, id) {
 441                 if (id >= err_id) {
 442                         kfree(engine);
 443                         dev_priv->engine[id] = NULL;
 444                 } else {
 445                         dev_priv->gt.cleanup_engine(engine);
 446                 }
 447         }
 448         return err;
 449 }
 450
 451 static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 452 {
 453         i915_gem_batch_pool_init(&engine->batch_pool, engine);
 454 }
 455
 456 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 457 {
 458         struct intel_engine_execlists * const execlists = &engine->execlists;
 459
 460         execlists->port_mask = 1;
 461         GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
 462         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 463
 464         execlists->queue_priority_hint = INT_MIN;
 465         execlists->queue = RB_ROOT_CACHED;
 466 }
 467
 468 static void cleanup_status_page(struct intel_engine_cs *engine)
 469 {
 470         struct i915_vma *vma;
 471
 472         /* Prevent writes into HWSP after returning the page to the system */
 473         intel_engine_set_hwsp_writemask(engine, ~0u);
 474
 475         vma = fetch_and_zero(&engine->status_page.vma);
 476         if (!vma)
 477                 return;
 478
 479         if (!HWS_NEEDS_PHYSICAL(engine->i915))
 480                 i915_vma_unpin(vma);
 481
 482         i915_gem_object_unpin_map(vma->obj);
 483         __i915_gem_object_release_unless_active(vma->obj);
 484 }
 485
 486 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
 487                                 struct i915_vma *vma)
 488 {
 489         unsigned int flags;
 490
 491         flags = PIN_GLOBAL;
 492         if (!HAS_LLC(engine->i915))
 493                 /*
 494                  * On g33, we cannot place HWS above 256MiB, so
 495                  * restrict its pinning to the low mappable arena.
 496                  * Though this restriction is not documented for
 497                  * gen4, gen5, or byt, they also behave similarly
 498                  * and hang if the HWS is placed at the top of the
 499                  * GTT. To generalise, it appears that all !llc
 500                  * platforms have issues with us placing the HWS
 501                  * above the mappable region (even though we never
 502                  * actually map it).
 503                  */
 504                 flags |= PIN_MAPPABLE;
 505         else
 506                 flags |= PIN_HIGH;
 507
 508         return i915_vma_pin(vma, 0, 0, flags);
 509 }
 510
 511 static int init_status_page(struct intel_engine_cs *engine)
 512 {
 513         struct drm_i915_gem_object *obj;
 514         struct i915_vma *vma;
 515         void *vaddr;
 516         int ret;
 517
 518         /*
 519          * Though the HWS register does support 36bit addresses, historically
 520          * we have had hangs and corruption reported due to wild writes if
 521          * the HWS is placed above 4G. We only allow objects to be allocated
 522          * in GFP_DMA32 for i965, and no earlier physical address users had
 523          * access to more than 4G.
 524          */
 525         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 526         if (IS_ERR(obj)) {
 527                 DRM_ERROR("Failed to allocate status page\n");
 528                 return PTR_ERR(obj);
 529         }
 530
 531         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 532
 533         vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
 534         if (IS_ERR(vma)) {
 535                 ret = PTR_ERR(vma);
 536                 goto err;
 537         }
 538
 539         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 540         if (IS_ERR(vaddr)) {
 541                 ret = PTR_ERR(vaddr);
 542                 goto err;
 543         }
 544
 545         engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
 546         engine->status_page.vma = vma;
 547
 548         if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
 549                 ret = pin_ggtt_status_page(engine, vma);
 550                 if (ret)
 551                         goto err_unpin;
 552         }
 553
 554         return 0;
 555
 556 err_unpin:
 557         i915_gem_object_unpin_map(obj);
 558 err:
 559         i915_gem_object_put(obj);
 560         return ret;
 561 }
 562
 563 /**
 564  * intel_engines_setup_common - setup engine state not requiring hw access
 565  * @engine: Engine to setup.
 566  *
 567  * Initializes @engine@ structure members shared between legacy and execlists
 568  * submission modes which do not require hardware access.
 569  *
 570  * Typically done early in the submission mode specific engine setup stage.
 571  */
 572 int intel_engine_setup_common(struct intel_engine_cs *engine)
 573 {
 574         int err;
 575
 576         err = init_status_page(engine);
 577         if (err)
 578                 return err;
 579
 580         err = i915_timeline_init(engine->i915,
 581                                  &engine->timeline,
 582                                  engine->status_page.vma);
 583         if (err)
 584                 goto err_hwsp;
 585
 586         i915_timeline_set_subclass(&engine->timeline, TIMELINE_ENGINE);
 587
 588         intel_engine_init_breadcrumbs(engine);
 589         intel_engine_init_execlist(engine);
 590         intel_engine_init_hangcheck(engine);
 591         intel_engine_init_batch_pool(engine);
 592         intel_engine_init_cmd_parser(engine);
 593
 594         return 0;
 595
 596 err_hwsp:
 597         cleanup_status_page(engine);
 598         return err;
 599 }
 600
 601 void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
 602 {
 603         static const struct {
 604                 u8 engine;
 605                 u8 sched;
 606         } map[] = {
 607 #define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
 608                 MAP(PREEMPTION, PREEMPTION),
 609                 MAP(SEMAPHORES, SEMAPHORES),
 610 #undef MAP
 611         };
 612         struct intel_engine_cs *engine;
 613         enum intel_engine_id id;
 614         u32 enabled, disabled;
 615
 616         enabled = 0;
 617         disabled = 0;
 618         for_each_engine(engine, i915, id) { /* all engines must agree! */
 619                 int i;
 620
 621                 if (engine->schedule)
 622                         enabled |= (I915_SCHEDULER_CAP_ENABLED |
 623                                     I915_SCHEDULER_CAP_PRIORITY);
 624                 else
 625                         disabled |= (I915_SCHEDULER_CAP_ENABLED |
 626                                      I915_SCHEDULER_CAP_PRIORITY);
 627
 628                 for (i = 0; i < ARRAY_SIZE(map); i++) {
 629                         if (engine->flags & BIT(map[i].engine))
 630                                 enabled |= BIT(map[i].sched);
 631                         else
 632                                 disabled |= BIT(map[i].sched);
 633                 }
 634         }
 635
 636         i915->caps.scheduler = enabled & ~disabled;
 637         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
 638                 i915->caps.scheduler = 0;
 639 }
 640
 641 struct measure_breadcrumb {
 642         struct i915_request rq;
 643         struct i915_timeline timeline;
 644         struct intel_ring ring;
 645         u32 cs[1024];
 646 };
 647
 648 static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
 649 {
 650         struct measure_breadcrumb *frame;
 651         int dw = -ENOMEM;
 652
 653         GEM_BUG_ON(!engine->i915->gt.scratch);
 654
 655         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 656         if (!frame)
 657                 return -ENOMEM;
 658
 659         if (i915_timeline_init(engine->i915,
 660                                &frame->timeline,
 661                                engine->status_page.vma))
 662                 goto out_frame;
 663
 664         INIT_LIST_HEAD(&frame->ring.request_list);
 665         frame->ring.timeline = &frame->timeline;
 666         frame->ring.vaddr = frame->cs;
 667         frame->ring.size = sizeof(frame->cs);
 668         frame->ring.effective_size = frame->ring.size;
 669         intel_ring_update_space(&frame->ring);
 670
 671         frame->rq.i915 = engine->i915;
 672         frame->rq.engine = engine;
 673         frame->rq.ring = &frame->ring;
 674         frame->rq.timeline = &frame->timeline;
 675
 676         dw = i915_timeline_pin(&frame->timeline);
 677         if (dw < 0)
 678                 goto out_timeline;
 679
 680         dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
 681
 682         i915_timeline_unpin(&frame->timeline);
 683
 684 out_timeline:
 685         i915_timeline_fini(&frame->timeline);
 686 out_frame:
 687         kfree(frame);
 688         return dw;
 689 }
 690
 691 static int pin_context(struct i915_gem_context *ctx,
 692                        struct intel_engine_cs *engine,
 693                        struct intel_context **out)
 694 {
 695         struct intel_context *ce;
 696
 697         ce = intel_context_pin(ctx, engine);
 698         if (IS_ERR(ce))
 699                 return PTR_ERR(ce);
 700
 701         *out = ce;
 702         return 0;
 703 }
 704
 705 /**
 706  * intel_engines_init_common - initialize cengine state which might require hw access
 707  * @engine: Engine to initialize.
 708  *
 709  * Initializes @engine@ structure members shared between legacy and execlists
 710  * submission modes which do require hardware access.
 711  *
 712  * Typcally done at later stages of submission mode specific engine setup.
 713  *
 714  * Returns zero on success or an error code on failure.
 715  */
 716 int intel_engine_init_common(struct intel_engine_cs *engine)
 717 {
 718         struct drm_i915_private *i915 = engine->i915;
 719         int ret;
 720
 721         /* We may need to do things with the shrinker which
 722          * require us to immediately switch back to the default
 723          * context. This can cause a problem as pinning the
 724          * default context also requires GTT space which may not
 725          * be available. To avoid this we always pin the default
 726          * context.
 727          */
 728         ret = pin_context(i915->kernel_context, engine,
 729                           &engine->kernel_context);
 730         if (ret)
 731                 return ret;
 732
 733         /*
 734          * Similarly the preempt context must always be available so that
 735          * we can interrupt the engine at any time. However, as preemption
 736          * is optional, we allow it to fail.
 737          */
 738         if (i915->preempt_context)
 739                 pin_context(i915->preempt_context, engine,
 740                             &engine->preempt_context);
 741
 742         ret = measure_breadcrumb_dw(engine);
 743         if (ret < 0)
 744                 goto err_unpin;
 745
 746         engine->emit_fini_breadcrumb_dw = ret;
 747
 748         engine->set_default_submission(engine);
 749
 750         return 0;
 751
 752 err_unpin:
 753         if (engine->preempt_context)
 754                 intel_context_unpin(engine->preempt_context);
 755         intel_context_unpin(engine->kernel_context);
 756         return ret;
 757 }
 758
 759 /**
 760  * intel_engines_cleanup_common - cleans up the engine state created by
 761  *                                the common initiailizers.
 762  * @engine: Engine to cleanup.
 763  *
 764  * This cleans up everything created by the common helpers.
 765  */
 766 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 767 {
 768         cleanup_status_page(engine);
 769
 770         intel_engine_fini_breadcrumbs(engine);
 771         intel_engine_cleanup_cmd_parser(engine);
 772         i915_gem_batch_pool_fini(&engine->batch_pool);
 773
 774         if (engine->default_state)
 775                 i915_gem_object_put(engine->default_state);
 776
 777         if (engine->preempt_context)
 778                 intel_context_unpin(engine->preempt_context);
 779         intel_context_unpin(engine->kernel_context);
 780
 781         i915_timeline_fini(&engine->timeline);
 782
 783         intel_wa_list_free(&engine->ctx_wa_list);
 784         intel_wa_list_free(&engine->wa_list);
 785         intel_wa_list_free(&engine->whitelist);
 786 }
 787
 788 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
 789 {
 790         struct drm_i915_private *dev_priv = engine->i915;
 791         u64 acthd;
 792
 793         if (INTEL_GEN(dev_priv) >= 8)
 794                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
 795                                          RING_ACTHD_UDW(engine->mmio_base));
 796         else if (INTEL_GEN(dev_priv) >= 4)
 797                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
 798         else
 799                 acthd = I915_READ(ACTHD);
 800
 801         return acthd;
 802 }
 803
 804 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 805 {
 806         struct drm_i915_private *dev_priv = engine->i915;
 807         u64 bbaddr;
 808
 809         if (INTEL_GEN(dev_priv) >= 8)
 810                 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
 811                                           RING_BBADDR_UDW(engine->mmio_base));
 812         else
 813                 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
 814
 815         return bbaddr;
 816 }
 817
 818 int intel_engine_stop_cs(struct intel_engine_cs *engine)
 819 {
 820         struct drm_i915_private *dev_priv = engine->i915;
 821         const u32 base = engine->mmio_base;
 822         const i915_reg_t mode = RING_MI_MODE(base);
 823         int err;
 824
 825         if (INTEL_GEN(dev_priv) < 3)
 826                 return -ENODEV;
 827
 828         GEM_TRACE("%s\n", engine->name);
 829
 830         I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
 831
 832         err = 0;
 833         if (__intel_wait_for_register_fw(dev_priv,
 834                                          mode, MODE_IDLE, MODE_IDLE,
 835                                          1000, 0,
 836                                          NULL)) {
 837                 GEM_TRACE("%s: timed out on STOP_RING -> IDLE\n", engine->name);
 838                 err = -ETIMEDOUT;
 839         }
 840
 841         /* A final mmio read to let GPU writes be hopefully flushed to memory */
 842         POSTING_READ_FW(mode);
 843
 844         return err;
 845 }
 846
 847 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 848 {
 849         struct drm_i915_private *dev_priv = engine->i915;
 850
 851         GEM_TRACE("%s\n", engine->name);
 852
 853         I915_WRITE_FW(RING_MI_MODE(engine->mmio_base),
 854                       _MASKED_BIT_DISABLE(STOP_RING));
 855 }
 856
 857 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 858 {
 859         switch (type) {
 860         case I915_CACHE_NONE: return " uncached";
 861         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
 862         case I915_CACHE_L3_LLC: return " L3+LLC";
 863         case I915_CACHE_WT: return " WT";
 864         default: return "";
 865         }
 866 }
 867
 868 u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv)
 869 {
 870         const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 871         u32 mcr_s_ss_select;
 872         u32 slice = fls(sseu->slice_mask);
 873         u32 subslice = fls(sseu->subslice_mask[slice]);
 874
 875         if (IS_GEN(dev_priv, 10))
 876                 mcr_s_ss_select = GEN8_MCR_SLICE(slice) |
 877                                   GEN8_MCR_SUBSLICE(subslice);
 878         else if (INTEL_GEN(dev_priv) >= 11)
 879                 mcr_s_ss_select = GEN11_MCR_SLICE(slice) |
 880                                   GEN11_MCR_SUBSLICE(subslice);
 881         else
 882                 mcr_s_ss_select = 0;
 883
 884         return mcr_s_ss_select;
 885 }
 886
 887 static inline u32
 888 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 889                   int subslice, i915_reg_t reg)
 890 {
 891         struct intel_uncore *uncore = &dev_priv->uncore;
 892         u32 mcr_slice_subslice_mask;
 893         u32 mcr_slice_subslice_select;
 894         u32 default_mcr_s_ss_select;
 895         u32 mcr;
 896         u32 ret;
 897         enum forcewake_domains fw_domains;
 898
 899         if (INTEL_GEN(dev_priv) >= 11) {
 900                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
 901                                           GEN11_MCR_SUBSLICE_MASK;
 902                 mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
 903                                             GEN11_MCR_SUBSLICE(subslice);
 904         } else {
 905                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
 906                                           GEN8_MCR_SUBSLICE_MASK;
 907                 mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
 908                                             GEN8_MCR_SUBSLICE(subslice);
 909         }
 910
 911         default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(dev_priv);
 912
 913         fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
 914                                                     FW_REG_READ);
 915         fw_domains |= intel_uncore_forcewake_for_reg(uncore,
 916                                                      GEN8_MCR_SELECTOR,
 917                                                      FW_REG_READ | FW_REG_WRITE);
 918
 919         spin_lock_irq(&uncore->lock);
 920         intel_uncore_forcewake_get__locked(uncore, fw_domains);
 921
 922         mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
 923
 924         WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
 925                      default_mcr_s_ss_select);
 926
 927         mcr &= ~mcr_slice_subslice_mask;
 928         mcr |= mcr_slice_subslice_select;
 929         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
 930
 931         ret = intel_uncore_read_fw(uncore, reg);
 932
 933         mcr &= ~mcr_slice_subslice_mask;
 934         mcr |= default_mcr_s_ss_select;
 935
 936         intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
 937
 938         intel_uncore_forcewake_put__locked(uncore, fw_domains);
 939         spin_unlock_irq(&uncore->lock);
 940
 941         return ret;
 942 }
 943
 944 /* NB: please notice the memset */
 945 void intel_engine_get_instdone(struct intel_engine_cs *engine,
 946                                struct intel_instdone *instdone)
 947 {
 948         struct drm_i915_private *dev_priv = engine->i915;
 949         u32 mmio_base = engine->mmio_base;
 950         int slice;
 951         int subslice;
 952
 953         memset(instdone, 0, sizeof(*instdone));
 954
 955         switch (INTEL_GEN(dev_priv)) {
 956         default:
 957                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 958
 959                 if (engine->id != RCS0)
 960                         break;
 961
 962                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 963                 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
 964                         instdone->sampler[slice][subslice] =
 965                                 read_subslice_reg(dev_priv, slice, subslice,
 966                                                   GEN7_SAMPLER_INSTDONE);
 967                         instdone->row[slice][subslice] =
 968                                 read_subslice_reg(dev_priv, slice, subslice,
 969                                                   GEN7_ROW_INSTDONE);
 970                 }
 971                 break;
 972         case 7:
 973                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 974
 975                 if (engine->id != RCS0)
 976                         break;
 977
 978                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 979                 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
 980                 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
 981
 982                 break;
 983         case 6:
 984         case 5:
 985         case 4:
 986                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 987
 988                 if (engine->id == RCS0)
 989                         /* HACK: Using the wrong struct member */
 990                         instdone->slice_common = I915_READ(GEN4_INSTDONE1);
 991                 break;
 992         case 3:
 993         case 2:
 994                 instdone->instdone = I915_READ(GEN2_INSTDONE);
 995                 break;
 996         }
 997 }
 998
 999 static bool ring_is_idle(struct intel_engine_cs *engine)
1000 {
1001         struct drm_i915_private *dev_priv = engine->i915;
1002         intel_wakeref_t wakeref;
1003         bool idle = true;
1004
1005         if (I915_SELFTEST_ONLY(!engine->mmio_base))
1006                 return true;
1007
1008         /* If the whole device is asleep, the engine must be idle */
1009         wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
1010         if (!wakeref)
1011                 return true;
1012
1013         /* First check that no commands are left in the ring */
1014         if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1015             (I915_READ_TAIL(engine) & TAIL_ADDR))
1016                 idle = false;
1017
1018         /* No bit for gen2, so assume the CS parser is idle */
1019         if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1020                 idle = false;
1021
1022         intel_runtime_pm_put(dev_priv, wakeref);
1023
1024         return idle;
1025 }
1026
1027 /**
1028  * intel_engine_is_idle() - Report if the engine has finished process all work
1029  * @engine: the intel_engine_cs
1030  *
1031  * Return true if there are no requests pending, nothing left to be submitted
1032  * to hardware, and that the engine is idle.
1033  */
1034 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1035 {
1036         /* More white lies, if wedged, hw state is inconsistent */
1037         if (i915_reset_failed(engine->i915))
1038                 return true;
1039
1040         /* Waiting to drain ELSP? */
1041         if (READ_ONCE(engine->execlists.active)) {
1042                 struct tasklet_struct *t = &engine->execlists.tasklet;
1043
1044                 local_bh_disable();
1045                 if (tasklet_trylock(t)) {
1046                         /* Must wait for any GPU reset in progress. */
1047                         if (__tasklet_is_enabled(t))
1048                                 t->func(t->data);
1049                         tasklet_unlock(t);
1050                 }
1051                 local_bh_enable();
1052
1053                 /* Otherwise flush the tasklet if it was on another cpu */
1054                 tasklet_unlock_wait(t);
1055
1056                 if (READ_ONCE(engine->execlists.active))
1057                         return false;
1058         }
1059
1060         /* ELSP is empty, but there are ready requests? E.g. after reset */
1061         if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
1062                 return false;
1063
1064         /* Ring stopped? */
1065         return ring_is_idle(engine);
1066 }
1067
1068 bool intel_engines_are_idle(struct drm_i915_private *i915)
1069 {
1070         struct intel_engine_cs *engine;
1071         enum intel_engine_id id;
1072
1073         /*
1074          * If the driver is wedged, HW state may be very inconsistent and
1075          * report that it is still busy, even though we have stopped using it.
1076          */
1077         if (i915_reset_failed(i915))
1078                 return true;
1079
1080         /* Already parked (and passed an idleness test); must still be idle */
1081         if (!READ_ONCE(i915->gt.awake))
1082                 return true;
1083
1084         for_each_engine(engine, i915, id) {
1085                 if (!intel_engine_is_idle(engine))
1086                         return false;
1087         }
1088
1089         return true;
1090 }
1091
1092 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1093 {
1094         struct intel_engine_cs *engine;
1095         enum intel_engine_id id;
1096
1097         for_each_engine(engine, i915, id)
1098                 engine->set_default_submission(engine);
1099 }
1100
1101 static bool reset_engines(struct drm_i915_private *i915)
1102 {
1103         if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
1104                 return false;
1105
1106         return intel_gpu_reset(i915, ALL_ENGINES) == 0;
1107 }
1108
1109 /**
1110  * intel_engines_sanitize: called after the GPU has lost power
1111  * @i915: the i915 device
1112  * @force: ignore a failed reset and sanitize engine state anyway
1113  *
1114  * Anytime we reset the GPU, either with an explicit GPU reset or through a
1115  * PCI power cycle, the GPU loses state and we must reset our state tracking
1116  * to match. Note that calling intel_engines_sanitize() if the GPU has not
1117  * been reset results in much confusion!
1118  */
1119 void intel_engines_sanitize(struct drm_i915_private *i915, bool force)
1120 {
1121         struct intel_engine_cs *engine;
1122         enum intel_engine_id id;
1123
1124         GEM_TRACE("\n");
1125
1126         if (!reset_engines(i915) && !force)
1127                 return;
1128
1129         for_each_engine(engine, i915, id)
1130                 intel_engine_reset(engine, false);
1131 }
1132
1133 /**
1134  * intel_engines_park: called when the GT is transitioning from busy->idle
1135  * @i915: the i915 device
1136  *
1137  * The GT is now idle and about to go to sleep (maybe never to wake again?).
1138  * Time for us to tidy and put away our toys (release resources back to the
1139  * system).
1140  */
1141 void intel_engines_park(struct drm_i915_private *i915)
1142 {
1143         struct intel_engine_cs *engine;
1144         enum intel_engine_id id;
1145
1146         for_each_engine(engine, i915, id) {
1147                 /* Flush the residual irq tasklets first. */
1148                 intel_engine_disarm_breadcrumbs(engine);
1149                 tasklet_kill(&engine->execlists.tasklet);
1150
1151                 /*
1152                  * We are committed now to parking the engines, make sure there
1153                  * will be no more interrupts arriving later and the engines
1154                  * are truly idle.
1155                  */
1156                 if (wait_for(intel_engine_is_idle(engine), 10)) {
1157                         struct drm_printer p = drm_debug_printer(__func__);
1158
1159                         dev_err(i915->drm.dev,
1160                                 "%s is not idle before parking\n",
1161                                 engine->name);
1162                         intel_engine_dump(engine, &p, NULL);
1163                 }
1164
1165                 /* Must be reset upon idling, or we may miss the busy wakeup. */
1166                 GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
1167
1168                 if (engine->park)
1169                         engine->park(engine);
1170
1171                 if (engine->pinned_default_state) {
1172                         i915_gem_object_unpin_map(engine->default_state);
1173                         engine->pinned_default_state = NULL;
1174                 }
1175
1176                 i915_gem_batch_pool_fini(&engine->batch_pool);
1177                 engine->execlists.no_priolist = false;
1178         }
1179
1180         i915->gt.active_engines = 0;
1181 }
1182
1183 /**
1184  * intel_engines_unpark: called when the GT is transitioning from idle->busy
1185  * @i915: the i915 device
1186  *
1187  * The GT was idle and now about to fire up with some new user requests.
1188  */
1189 void intel_engines_unpark(struct drm_i915_private *i915)
1190 {
1191         struct intel_engine_cs *engine;
1192         enum intel_engine_id id;
1193
1194         for_each_engine(engine, i915, id) {
1195                 void *map;
1196
1197                 /* Pin the default state for fast resets from atomic context. */
1198                 map = NULL;
1199                 if (engine->default_state)
1200                         map = i915_gem_object_pin_map(engine->default_state,
1201                                                       I915_MAP_WB);
1202                 if (!IS_ERR_OR_NULL(map))
1203                         engine->pinned_default_state = map;
1204
1205                 if (engine->unpark)
1206                         engine->unpark(engine);
1207
1208                 intel_engine_init_hangcheck(engine);
1209         }
1210 }
1211
1212 /**
1213  * intel_engine_lost_context: called when the GPU is reset into unknown state
1214  * @engine: the engine
1215  *
1216  * We have either reset the GPU or otherwise about to lose state tracking of
1217  * the current GPU logical state (e.g. suspend). On next use, it is therefore
1218  * imperative that we make no presumptions about the current state and load
1219  * from scratch.
1220  */
1221 void intel_engine_lost_context(struct intel_engine_cs *engine)
1222 {
1223         struct intel_context *ce;
1224
1225         lockdep_assert_held(&engine->i915->drm.struct_mutex);
1226
1227         ce = fetch_and_zero(&engine->last_retired_context);
1228         if (ce)
1229                 intel_context_unpin(ce);
1230 }
1231
1232 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1233 {
1234         switch (INTEL_GEN(engine->i915)) {
1235         case 2:
1236                 return false; /* uses physical not virtual addresses */
1237         case 3:
1238                 /* maybe only uses physical not virtual addresses */
1239                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1240         case 6:
1241                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1242         default:
1243                 return true;
1244         }
1245 }
1246
1247 unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1248 {
1249         struct intel_engine_cs *engine;
1250         enum intel_engine_id id;
1251         unsigned int which;
1252
1253         which = 0;
1254         for_each_engine(engine, i915, id)
1255                 if (engine->default_state)
1256                         which |= BIT(engine->uabi_class);
1257
1258         return which;
1259 }
1260
1261 static int print_sched_attr(struct drm_i915_private *i915,
1262                             const struct i915_sched_attr *attr,
1263                             char *buf, int x, int len)
1264 {
1265         if (attr->priority == I915_PRIORITY_INVALID)
1266                 return x;
1267
1268         x += snprintf(buf + x, len - x,
1269                       " prio=%d", attr->priority);
1270
1271         return x;
1272 }
1273
1274 static void print_request(struct drm_printer *m,
1275                           struct i915_request *rq,
1276                           const char *prefix)
1277 {
1278         const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1279         char buf[80] = "";
1280         int x = 0;
1281
1282         x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
1283
1284         drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1285                    prefix,
1286                    rq->fence.context, rq->fence.seqno,
1287                    i915_request_completed(rq) ? "!" :
1288                    i915_request_started(rq) ? "*" :
1289                    "",
1290                    test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1291                             &rq->fence.flags) ?  "+" : "",
1292                    buf,
1293                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1294                    name);
1295 }
1296
1297 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1298 {
1299         const size_t rowsize = 8 * sizeof(u32);
1300         const void *prev = NULL;
1301         bool skip = false;
1302         size_t pos;
1303
1304         for (pos = 0; pos < len; pos += rowsize) {
1305                 char line[128];
1306
1307                 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1308                         if (!skip) {
1309                                 drm_printf(m, "*\n");
1310                                 skip = true;
1311                         }
1312                         continue;
1313                 }
1314
1315                 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1316                                                 rowsize, sizeof(u32),
1317                                                 line, sizeof(line),
1318                                                 false) >= sizeof(line));
1319                 drm_printf(m, "[%04zx] %s\n", pos, line);
1320
1321                 prev = buf + pos;
1322                 skip = false;
1323         }
1324 }
1325
1326 static void intel_engine_print_registers(const struct intel_engine_cs *engine,
1327                                          struct drm_printer *m)
1328 {
1329         struct drm_i915_private *dev_priv = engine->i915;
1330         const struct intel_engine_execlists * const execlists =
1331                 &engine->execlists;
1332         u64 addr;
1333
1334         if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
1335                 drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID));
1336         drm_printf(m, "\tRING_START: 0x%08x\n",
1337                    I915_READ(RING_START(engine->mmio_base)));
1338         drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1339                    I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR);
1340         drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1341                    I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR);
1342         drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1343                    I915_READ(RING_CTL(engine->mmio_base)),
1344                    I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1345         if (INTEL_GEN(engine->i915) > 2) {
1346                 drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1347                            I915_READ(RING_MI_MODE(engine->mmio_base)),
1348                            I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
1349         }
1350
1351         if (INTEL_GEN(dev_priv) >= 6) {
1352                 drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine));
1353         }
1354
1355         addr = intel_engine_get_active_head(engine);
1356         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1357                    upper_32_bits(addr), lower_32_bits(addr));
1358         addr = intel_engine_get_last_batch_head(engine);
1359         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1360                    upper_32_bits(addr), lower_32_bits(addr));
1361         if (INTEL_GEN(dev_priv) >= 8)
1362                 addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base),
1363                                         RING_DMA_FADD_UDW(engine->mmio_base));
1364         else if (INTEL_GEN(dev_priv) >= 4)
1365                 addr = I915_READ(RING_DMA_FADD(engine->mmio_base));
1366         else
1367                 addr = I915_READ(DMA_FADD_I8XX);
1368         drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1369                    upper_32_bits(addr), lower_32_bits(addr));
1370         if (INTEL_GEN(dev_priv) >= 4) {
1371                 drm_printf(m, "\tIPEIR: 0x%08x\n",
1372                            I915_READ(RING_IPEIR(engine->mmio_base)));
1373                 drm_printf(m, "\tIPEHR: 0x%08x\n",
1374                            I915_READ(RING_IPEHR(engine->mmio_base)));
1375         } else {
1376                 drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR));
1377                 drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR));
1378         }
1379
1380         if (HAS_EXECLISTS(dev_priv)) {
1381                 const u32 *hws =
1382                         &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1383                 unsigned int idx;
1384                 u8 read, write;
1385
1386                 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1387                            I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1388                            I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1389
1390                 read = execlists->csb_head;
1391                 write = READ_ONCE(*execlists->csb_write);
1392
1393                 drm_printf(m, "\tExeclist CSB read %d, write %d [mmio:%d], tasklet queued? %s (%s)\n",
1394                            read, write,
1395                            GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine))),
1396                            yesno(test_bit(TASKLET_STATE_SCHED,
1397                                           &engine->execlists.tasklet.state)),
1398                            enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
1399                 if (read >= GEN8_CSB_ENTRIES)
1400                         read = 0;
1401                 if (write >= GEN8_CSB_ENTRIES)
1402                         write = 0;
1403                 if (read > write)
1404                         write += GEN8_CSB_ENTRIES;
1405                 while (read < write) {
1406                         idx = ++read % GEN8_CSB_ENTRIES;
1407                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [mmio:0x%08x], context: %d [mmio:%d]\n",
1408                                    idx,
1409                                    hws[idx * 2],
1410                                    I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1411                                    hws[idx * 2 + 1],
1412                                    I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
1413                 }
1414
1415                 rcu_read_lock();
1416                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1417                         struct i915_request *rq;
1418                         unsigned int count;
1419
1420                         rq = port_unpack(&execlists->port[idx], &count);
1421                         if (rq) {
1422                                 char hdr[80];
1423
1424                                 snprintf(hdr, sizeof(hdr),
1425                                          "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
1426                                          idx, count,
1427                                          i915_ggtt_offset(rq->ring->vma),
1428                                          rq->timeline->hwsp_offset,
1429                                          hwsp_seqno(rq));
1430                                 print_request(m, rq, hdr);
1431                         } else {
1432                                 drm_printf(m, "\t\tELSP[%d] idle\n", idx);
1433                         }
1434                 }
1435                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1436                 rcu_read_unlock();
1437         } else if (INTEL_GEN(dev_priv) > 6) {
1438                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1439                            I915_READ(RING_PP_DIR_BASE(engine)));
1440                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1441                            I915_READ(RING_PP_DIR_BASE_READ(engine)));
1442                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1443                            I915_READ(RING_PP_DIR_DCLV(engine)));
1444         }
1445 }
1446
1447 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1448 {
1449         void *ring;
1450         int size;
1451
1452         drm_printf(m,
1453                    "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1454                    rq->head, rq->postfix, rq->tail,
1455                    rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1456                    rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1457
1458         size = rq->tail - rq->head;
1459         if (rq->tail < rq->head)
1460                 size += rq->ring->size;
1461
1462         ring = kmalloc(size, GFP_ATOMIC);
1463         if (ring) {
1464                 const void *vaddr = rq->ring->vaddr;
1465                 unsigned int head = rq->head;
1466                 unsigned int len = 0;
1467
1468                 if (rq->tail < head) {
1469                         len = rq->ring->size - head;
1470                         memcpy(ring, vaddr + head, len);
1471                         head = 0;
1472                 }
1473                 memcpy(ring + len, vaddr + head, size - len);
1474
1475                 hexdump(m, ring, size);
1476                 kfree(ring);
1477         }
1478 }
1479
1480 void intel_engine_dump(struct intel_engine_cs *engine,
1481                        struct drm_printer *m,
1482                        const char *header, ...)
1483 {
1484         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1485         struct i915_request *rq;
1486         intel_wakeref_t wakeref;
1487
1488         if (header) {
1489                 va_list ap;
1490
1491                 va_start(ap, header);
1492                 drm_vprintf(m, header, &ap);
1493                 va_end(ap);
1494         }
1495
1496         if (i915_reset_failed(engine->i915))
1497                 drm_printf(m, "*** WEDGED ***\n");
1498
1499         drm_printf(m, "\tHangcheck %x:%x [%d ms]\n",
1500                    engine->hangcheck.last_seqno,
1501                    engine->hangcheck.next_seqno,
1502                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
1503         drm_printf(m, "\tReset count: %d (global %d)\n",
1504                    i915_reset_engine_count(error, engine),
1505                    i915_reset_count(error));
1506
1507         rcu_read_lock();
1508
1509         drm_printf(m, "\tRequests:\n");
1510
1511         rq = list_first_entry(&engine->timeline.requests,
1512                               struct i915_request, link);
1513         if (&rq->link != &engine->timeline.requests)
1514                 print_request(m, rq, "\t\tfirst  ");
1515
1516         rq = list_last_entry(&engine->timeline.requests,
1517                              struct i915_request, link);
1518         if (&rq->link != &engine->timeline.requests)
1519                 print_request(m, rq, "\t\tlast   ");
1520
1521         rq = intel_engine_find_active_request(engine);
1522         if (rq) {
1523                 print_request(m, rq, "\t\tactive ");
1524
1525                 drm_printf(m, "\t\tring->start:  0x%08x\n",
1526                            i915_ggtt_offset(rq->ring->vma));
1527                 drm_printf(m, "\t\tring->head:   0x%08x\n",
1528                            rq->ring->head);
1529                 drm_printf(m, "\t\tring->tail:   0x%08x\n",
1530                            rq->ring->tail);
1531                 drm_printf(m, "\t\tring->emit:   0x%08x\n",
1532                            rq->ring->emit);
1533                 drm_printf(m, "\t\tring->space:  0x%08x\n",
1534                            rq->ring->space);
1535                 drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
1536                            rq->timeline->hwsp_offset);
1537
1538                 print_request_ring(m, rq);
1539         }
1540
1541         rcu_read_unlock();
1542
1543         wakeref = intel_runtime_pm_get_if_in_use(engine->i915);
1544         if (wakeref) {
1545                 intel_engine_print_registers(engine, m);
1546                 intel_runtime_pm_put(engine->i915, wakeref);
1547         } else {
1548                 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
1549         }
1550
1551         intel_execlists_show_requests(engine, m, print_request, 8);
1552
1553         drm_printf(m, "HWSP:\n");
1554         hexdump(m, engine->status_page.addr, PAGE_SIZE);
1555
1556         drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1557
1558         intel_engine_print_breadcrumbs(engine, m);
1559 }
1560
1561 static u8 user_class_map[] = {
1562         [I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
1563         [I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
1564         [I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
1565         [I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
1566 };
1567
1568 struct intel_engine_cs *
1569 intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
1570 {
1571         if (class >= ARRAY_SIZE(user_class_map))
1572                 return NULL;
1573
1574         class = user_class_map[class];
1575
1576         GEM_BUG_ON(class > MAX_ENGINE_CLASS);
1577
1578         if (instance > MAX_ENGINE_INSTANCE)
1579                 return NULL;
1580
1581         return i915->engine_class[class][instance];
1582 }
1583
1584 /**
1585  * intel_enable_engine_stats() - Enable engine busy tracking on engine
1586  * @engine: engine to enable stats collection
1587  *
1588  * Start collecting the engine busyness data for @engine.
1589  *
1590  * Returns 0 on success or a negative error code.
1591  */
1592 int intel_enable_engine_stats(struct intel_engine_cs *engine)
1593 {
1594         struct intel_engine_execlists *execlists = &engine->execlists;
1595         unsigned long flags;
1596         int err = 0;
1597
1598         if (!intel_engine_supports_stats(engine))
1599                 return -ENODEV;
1600
1601         spin_lock_irqsave(&engine->timeline.lock, flags);
1602         write_seqlock(&engine->stats.lock);
1603
1604         if (unlikely(engine->stats.enabled == ~0)) {
1605                 err = -EBUSY;
1606                 goto unlock;
1607         }
1608
1609         if (engine->stats.enabled++ == 0) {
1610                 const struct execlist_port *port = execlists->port;
1611                 unsigned int num_ports = execlists_num_ports(execlists);
1612
1613                 engine->stats.enabled_at = ktime_get();
1614
1615                 /* XXX submission method oblivious? */
1616                 while (num_ports-- && port_isset(port)) {
1617                         engine->stats.active++;
1618                         port++;
1619                 }
1620
1621                 if (engine->stats.active)
1622                         engine->stats.start = engine->stats.enabled_at;
1623         }
1624
1625 unlock:
1626         write_sequnlock(&engine->stats.lock);
1627         spin_unlock_irqrestore(&engine->timeline.lock, flags);
1628
1629         return err;
1630 }
1631
1632 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
1633 {
1634         ktime_t total = engine->stats.total;
1635
1636         /*
1637          * If the engine is executing something at the moment
1638          * add it to the total.
1639          */
1640         if (engine->stats.active)
1641                 total = ktime_add(total,
1642                                   ktime_sub(ktime_get(), engine->stats.start));
1643
1644         return total;
1645 }
1646
1647 /**
1648  * intel_engine_get_busy_time() - Return current accumulated engine busyness
1649  * @engine: engine to report on
1650  *
1651  * Returns accumulated time @engine was busy since engine stats were enabled.
1652  */
1653 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
1654 {
1655         unsigned int seq;
1656         ktime_t total;
1657
1658         do {
1659                 seq = read_seqbegin(&engine->stats.lock);
1660                 total = __intel_engine_get_busy_time(engine);
1661         } while (read_seqretry(&engine->stats.lock, seq));
1662
1663         return total;
1664 }
1665
1666 /**
1667  * intel_disable_engine_stats() - Disable engine busy tracking on engine
1668  * @engine: engine to disable stats collection
1669  *
1670  * Stops collecting the engine busyness data for @engine.
1671  */
1672 void intel_disable_engine_stats(struct intel_engine_cs *engine)
1673 {
1674         unsigned long flags;
1675
1676         if (!intel_engine_supports_stats(engine))
1677                 return;
1678
1679         write_seqlock_irqsave(&engine->stats.lock, flags);
1680         WARN_ON_ONCE(engine->stats.enabled == 0);
1681         if (--engine->stats.enabled == 0) {
1682                 engine->stats.total = __intel_engine_get_busy_time(engine);
1683                 engine->stats.active = 0;
1684         }
1685         write_sequnlock_irqrestore(&engine->stats.lock, flags);
1686 }
1687
1688 static bool match_ring(struct i915_request *rq)
1689 {
1690         struct drm_i915_private *dev_priv = rq->i915;
1691         u32 ring = I915_READ(RING_START(rq->engine->mmio_base));
1692
1693         return ring == i915_ggtt_offset(rq->ring->vma);
1694 }
1695
1696 struct i915_request *
1697 intel_engine_find_active_request(struct intel_engine_cs *engine)
1698 {
1699         struct i915_request *request, *active = NULL;
1700         unsigned long flags;
1701
1702         /*
1703          * We are called by the error capture, reset and to dump engine
1704          * state at random points in time. In particular, note that neither is
1705          * crucially ordered with an interrupt. After a hang, the GPU is dead
1706          * and we assume that no more writes can happen (we waited long enough
1707          * for all writes that were in transaction to be flushed) - adding an
1708          * extra delay for a recent interrupt is pointless. Hence, we do
1709          * not need an engine->irq_seqno_barrier() before the seqno reads.
1710          * At all other times, we must assume the GPU is still running, but
1711          * we only care about the snapshot of this moment.
1712          */
1713         spin_lock_irqsave(&engine->timeline.lock, flags);
1714         list_for_each_entry(request, &engine->timeline.requests, link) {
1715                 if (i915_request_completed(request))
1716                         continue;
1717
1718                 if (!i915_request_started(request))
1719                         break;
1720
1721                 /* More than one preemptible request may match! */
1722                 if (!match_ring(request))
1723                         break;
1724
1725                 active = request;
1726                 break;
1727         }
1728         spin_unlock_irqrestore(&engine->timeline.lock, flags);
1729
1730         return active;
1731 }
1732
1733 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1734 #include "selftests/mock_engine.c"
1735 #include "selftests/intel_engine_cs.c"
1736 #endif