drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  56 {
  57         wal->name = name;
  58         wal->engine_name = engine_name;
  59 }
  60
  61 #define WA_LIST_CHUNK (1 << 4)
  62
  63 static void wa_init_finish(struct i915_wa_list *wal)
  64 {
  65         /* Trim unused entries. */
  66         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  67                 struct i915_wa *list = kmemdup(wal->list,
  68                                                wal->count * sizeof(*list),
  69                                                GFP_KERNEL);
  70
  71                 if (list) {
  72                         kfree(wal->list);
  73                         wal->list = list;
  74                 }
  75         }
  76
  77         if (!wal->count)
  78                 return;
  79
  80         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
  81                          wal->wa_count, wal->name, wal->engine_name);
  82 }
  83
  84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  85 {
  86         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  87         unsigned int start = 0, end = wal->count;
  88         const unsigned int grow = WA_LIST_CHUNK;
  89         struct i915_wa *wa_;
  90
  91         GEM_BUG_ON(!is_power_of_2(grow));
  92
  93         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  94                 struct i915_wa *list;
  95
  96                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  97                                      GFP_KERNEL);
  98                 if (!list) {
  99                         DRM_ERROR("No space for workaround init!\n");
 100                         return;
 101                 }
 102
 103                 if (wal->list)
 104                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 105
 106                 wal->list = list;
 107         }
 108
 109         while (start < end) {
 110                 unsigned int mid = start + (end - start) / 2;
 111
 112                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 113                         start = mid + 1;
 114                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 115                         end = mid;
 116                 } else {
 117                         wa_ = &wal->list[mid];
 118
 119                         if ((wa->mask & ~wa_->mask) == 0) {
 120                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (mask: %08x, value: %08x)\n",
 121                                           i915_mmio_reg_offset(wa_->reg),
 122                                           wa_->mask, wa_->val);
 123
 124                                 wa_->val &= ~wa->mask;
 125                         }
 126
 127                         wal->wa_count++;
 128                         wa_->val |= wa->val;
 129                         wa_->mask |= wa->mask;
 130                         wa_->read |= wa->read;
 131                         return;
 132                 }
 133         }
 134
 135         wal->wa_count++;
 136         wa_ = &wal->list[wal->count++];
 137         *wa_ = *wa;
 138
 139         while (wa_-- > wal->list) {
 140                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 141                            i915_mmio_reg_offset(wa_[1].reg));
 142                 if (i915_mmio_reg_offset(wa_[1].reg) >
 143                     i915_mmio_reg_offset(wa_[0].reg))
 144                         break;
 145
 146                 swap(wa_[1], wa_[0]);
 147         }
 148 }
 149
 150 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
 151                    u32 val, u32 read_mask)
 152 {
 153         struct i915_wa wa = {
 154                 .reg  = reg,
 155                 .mask = mask,
 156                 .val  = val,
 157                 .read = read_mask,
 158         };
 159
 160         _wa_add(wal, &wa);
 161 }
 162
 163 static void
 164 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
 165                    u32 val)
 166 {
 167         wa_add(wal, reg, mask, val, mask);
 168 }
 169
 170 static void
 171 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 172 {
 173         wa_write_masked_or(wal, reg, val, _MASKED_BIT_ENABLE(val));
 174 }
 175
 176 static void
 177 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 178 {
 179         wa_write_masked_or(wal, reg, ~0, val);
 180 }
 181
 182 static void
 183 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 184 {
 185         wa_write_masked_or(wal, reg, val, val);
 186 }
 187
 188 #define WA_SET_BIT_MASKED(addr, mask) \
 189         wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_ENABLE(mask))
 190
 191 #define WA_CLR_BIT_MASKED(addr, mask) \
 192         wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_DISABLE(mask))
 193
 194 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 195         wa_write_masked_or(wal, (addr), (mask), _MASKED_FIELD((mask), (value)))
 196
 197 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 198                                       struct i915_wa_list *wal)
 199 {
 200         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 201
 202         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 203         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 204
 205         /* WaDisablePartialInstShootdown:bdw,chv */
 206         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 207                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 208
 209         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 210          * workaround for for a possible hang in the unlikely event a TLB
 211          * invalidation occurs during a PSD flush.
 212          */
 213         /* WaForceEnableNonCoherent:bdw,chv */
 214         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 215         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 216                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 217                           HDC_FORCE_NON_COHERENT);
 218
 219         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 220          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 221          *  polygons in the same 8x4 pixel/sample area to be processed without
 222          *  stalling waiting for the earlier ones to write to Hierarchical Z
 223          *  buffer."
 224          *
 225          * This optimization is off by default for BDW and CHV; turn it on.
 226          */
 227         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 228
 229         /* Wa4x4STCOptimizationDisable:bdw,chv */
 230         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 231
 232         /*
 233          * BSpec recommends 8x4 when MSAA is used,
 234          * however in practice 16x4 seems fastest.
 235          *
 236          * Note that PS/WM thread counts depend on the WIZ hashing
 237          * disable bit, which we don't touch here, but it's good
 238          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 239          */
 240         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 241                             GEN6_WIZ_HASHING_MASK,
 242                             GEN6_WIZ_HASHING_16x4);
 243 }
 244
 245 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 246                                      struct i915_wa_list *wal)
 247 {
 248         struct drm_i915_private *i915 = engine->i915;
 249
 250         gen8_ctx_workarounds_init(engine, wal);
 251
 252         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 253         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 254
 255         /* WaDisableDopClockGating:bdw
 256          *
 257          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 258          * to disable EUTC clock gating.
 259          */
 260         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 261                           DOP_CLOCK_GATING_DISABLE);
 262
 263         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 264                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 265
 266         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 267                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 268                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 269                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 270                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 271 }
 272
 273 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 274                                      struct i915_wa_list *wal)
 275 {
 276         gen8_ctx_workarounds_init(engine, wal);
 277
 278         /* WaDisableThreadStallDopClockGating:chv */
 279         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 280
 281         /* Improve HiZ throughput on CHV. */
 282         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 283 }
 284
 285 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 286                                       struct i915_wa_list *wal)
 287 {
 288         struct drm_i915_private *i915 = engine->i915;
 289
 290         if (HAS_LLC(i915)) {
 291                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 292                  *
 293                  * Must match Display Engine. See
 294                  * WaCompressedResourceDisplayNewHashMode.
 295                  */
 296                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 297                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 298                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 299                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 300         }
 301
 302         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 303         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 304         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 305                           FLOW_CONTROL_ENABLE |
 306                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 307
 308         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 309         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 310         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 311                           GEN9_ENABLE_YV12_BUGFIX |
 312                           GEN9_ENABLE_GPGPU_PREEMPTION);
 313
 314         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 315         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 316         WA_SET_BIT_MASKED(CACHE_MODE_1,
 317                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 318                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 319
 320         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 321         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 322                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 323
 324         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 325         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 326                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 327                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 328
 329         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 330          * both tied to WaForceContextSaveRestoreNonCoherent
 331          * in some hsds for skl. We keep the tie for all gen9. The
 332          * documentation is a bit hazy and so we want to get common behaviour,
 333          * even though there is no clear evidence we would need both on kbl/bxt.
 334          * This area has been source of system hangs so we play it safe
 335          * and mimic the skl regardless of what bspec says.
 336          *
 337          * Use Force Non-Coherent whenever executing a 3D context. This
 338          * is a workaround for a possible hang in the unlikely event
 339          * a TLB invalidation occurs during a PSD flush.
 340          */
 341
 342         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 343         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 344                           HDC_FORCE_NON_COHERENT);
 345
 346         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 347         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
 348                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 349                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 350
 351         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 352         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 353
 354         /*
 355          * Supporting preemption with fine-granularity requires changes in the
 356          * batch buffer programming. Since we can't break old userspace, we
 357          * need to set our default preemption level to safe value. Userspace is
 358          * still able to use more fine-grained preemption levels, since in
 359          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 360          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 361          * not real HW workarounds, but merely a way to start using preemption
 362          * while maintaining old contract with userspace.
 363          */
 364
 365         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 366         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 367
 368         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 369         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 370                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 371                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 372
 373         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 374         if (IS_GEN9_LP(i915))
 375                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 376 }
 377
 378 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 379                                 struct i915_wa_list *wal)
 380 {
 381         struct drm_i915_private *i915 = engine->i915;
 382         u8 vals[3] = { 0, 0, 0 };
 383         unsigned int i;
 384
 385         for (i = 0; i < 3; i++) {
 386                 u8 ss;
 387
 388                 /*
 389                  * Only consider slices where one, and only one, subslice has 7
 390                  * EUs
 391                  */
 392                 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
 393                         continue;
 394
 395                 /*
 396                  * subslice_7eu[i] != 0 (because of the check above) and
 397                  * ss_max == 4 (maximum number of subslices possible per slice)
 398                  *
 399                  * ->    0 <= ss <= 3;
 400                  */
 401                 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
 402                 vals[i] = 3 - ss;
 403         }
 404
 405         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 406                 return;
 407
 408         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 409         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 410                             GEN9_IZ_HASHING_MASK(2) |
 411                             GEN9_IZ_HASHING_MASK(1) |
 412                             GEN9_IZ_HASHING_MASK(0),
 413                             GEN9_IZ_HASHING(2, vals[2]) |
 414                             GEN9_IZ_HASHING(1, vals[1]) |
 415                             GEN9_IZ_HASHING(0, vals[0]));
 416 }
 417
 418 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 419                                      struct i915_wa_list *wal)
 420 {
 421         gen9_ctx_workarounds_init(engine, wal);
 422         skl_tune_iz_hashing(engine, wal);
 423 }
 424
 425 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 426                                      struct i915_wa_list *wal)
 427 {
 428         gen9_ctx_workarounds_init(engine, wal);
 429
 430         /* WaDisableThreadStallDopClockGating:bxt */
 431         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 432                           STALL_DOP_GATING_DISABLE);
 433
 434         /* WaToEnableHwFixForPushConstHWBug:bxt */
 435         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 436                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 437 }
 438
 439 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 440                                      struct i915_wa_list *wal)
 441 {
 442         struct drm_i915_private *i915 = engine->i915;
 443
 444         gen9_ctx_workarounds_init(engine, wal);
 445
 446         /* WaToEnableHwFixForPushConstHWBug:kbl */
 447         if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 448                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 449                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 450
 451         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 452         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 453                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 454 }
 455
 456 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 457                                      struct i915_wa_list *wal)
 458 {
 459         gen9_ctx_workarounds_init(engine, wal);
 460
 461         /* WaToEnableHwFixForPushConstHWBug:glk */
 462         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 463                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 464 }
 465
 466 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 467                                      struct i915_wa_list *wal)
 468 {
 469         gen9_ctx_workarounds_init(engine, wal);
 470
 471         /* WaToEnableHwFixForPushConstHWBug:cfl */
 472         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 473                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 474
 475         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 476         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 477                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 478 }
 479
 480 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 481                                      struct i915_wa_list *wal)
 482 {
 483         struct drm_i915_private *i915 = engine->i915;
 484
 485         /* WaForceContextSaveRestoreNonCoherent:cnl */
 486         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 487                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 488
 489         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
 490         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 491                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
 492
 493         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 494         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 495                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 496
 497         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
 498         if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
 499                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 500                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
 501
 502         /* WaPushConstantDereferenceHoldDisable:cnl */
 503         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 504
 505         /* FtrEnableFastAnisoL1BankingFix:cnl */
 506         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 507
 508         /* WaDisable3DMidCmdPreemption:cnl */
 509         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 510
 511         /* WaDisableGPGPUMidCmdPreemption:cnl */
 512         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 513                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 514                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 515
 516         /* WaDisableEarlyEOT:cnl */
 517         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 518 }
 519
 520 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 521                                      struct i915_wa_list *wal)
 522 {
 523         struct drm_i915_private *i915 = engine->i915;
 524
 525         /* WaDisableBankHangMode:icl */
 526         wa_write(wal,
 527                  GEN8_L3CNTLREG,
 528                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 529                  GEN8_ERRDETBCTRL);
 530
 531         /* Wa_1604370585:icl (pre-prod)
 532          * Formerly known as WaPushConstantDereferenceHoldDisable
 533          */
 534         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 535                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 536                                   PUSH_CONSTANT_DEREF_DISABLE);
 537
 538         /* WaForceEnableNonCoherent:icl
 539          * This is not the same workaround as in early Gen9 platforms, where
 540          * lacking this could cause system hangs, but coherency performance
 541          * overhead is high and only a few compute workloads really need it
 542          * (the register is whitelisted in hardware now, so UMDs can opt in
 543          * for coherency if they have a good reason).
 544          */
 545         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 546
 547         /* Wa_2006611047:icl (pre-prod)
 548          * Formerly known as WaDisableImprovedTdlClkGating
 549          */
 550         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 551                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 552                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 553
 554         /* Wa_2006665173:icl (pre-prod) */
 555         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 556                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 557                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 558
 559         /* WaEnableFloatBlendOptimization:icl */
 560         wa_write_masked_or(wal,
 561                            GEN10_CACHE_MODE_SS,
 562                            0, /* write-only, so skip validation */
 563                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 564
 565         /* WaDisableGPGPUMidThreadPreemption:icl */
 566         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 567                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 568                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 569
 570         /* allow headerless messages for preemptible GPGPU context */
 571         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 572                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 573 }
 574
 575 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
 576                                      struct i915_wa_list *wal)
 577 {
 578         u32 val;
 579
 580         /* Wa_1409142259:tgl */
 581         WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 582                           GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 583
 584         /* Wa_1604555607:tgl */
 585         val = intel_uncore_read(engine->uncore, FF_MODE2);
 586         val &= ~FF_MODE2_TDS_TIMER_MASK;
 587         val |= FF_MODE2_TDS_TIMER_128;
 588         /*
 589          * FIXME: FF_MODE2 register is not readable till TGL B0. We can
 590          * enable verification of WA from the later steppings, which enables
 591          * the read of FF_MODE2.
 592          */
 593         wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK, val,
 594                IS_TGL_REVID(engine->i915, TGL_REVID_A0, TGL_REVID_A0) ? 0 :
 595                             FF_MODE2_TDS_TIMER_MASK);
 596 }
 597
 598 static void
 599 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 600                            struct i915_wa_list *wal,
 601                            const char *name)
 602 {
 603         struct drm_i915_private *i915 = engine->i915;
 604
 605         if (engine->class != RENDER_CLASS)
 606                 return;
 607
 608         wa_init_start(wal, name, engine->name);
 609
 610         if (IS_GEN(i915, 12))
 611                 tgl_ctx_workarounds_init(engine, wal);
 612         else if (IS_GEN(i915, 11))
 613                 icl_ctx_workarounds_init(engine, wal);
 614         else if (IS_CANNONLAKE(i915))
 615                 cnl_ctx_workarounds_init(engine, wal);
 616         else if (IS_COFFEELAKE(i915))
 617                 cfl_ctx_workarounds_init(engine, wal);
 618         else if (IS_GEMINILAKE(i915))
 619                 glk_ctx_workarounds_init(engine, wal);
 620         else if (IS_KABYLAKE(i915))
 621                 kbl_ctx_workarounds_init(engine, wal);
 622         else if (IS_BROXTON(i915))
 623                 bxt_ctx_workarounds_init(engine, wal);
 624         else if (IS_SKYLAKE(i915))
 625                 skl_ctx_workarounds_init(engine, wal);
 626         else if (IS_CHERRYVIEW(i915))
 627                 chv_ctx_workarounds_init(engine, wal);
 628         else if (IS_BROADWELL(i915))
 629                 bdw_ctx_workarounds_init(engine, wal);
 630         else if (INTEL_GEN(i915) < 8)
 631                 return;
 632         else
 633                 MISSING_CASE(INTEL_GEN(i915));
 634
 635         wa_init_finish(wal);
 636 }
 637
 638 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 639 {
 640         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 641 }
 642
 643 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 644 {
 645         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 646         struct i915_wa *wa;
 647         unsigned int i;
 648         u32 *cs;
 649         int ret;
 650
 651         if (wal->count == 0)
 652                 return 0;
 653
 654         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 655         if (ret)
 656                 return ret;
 657
 658         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 659         if (IS_ERR(cs))
 660                 return PTR_ERR(cs);
 661
 662         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 663         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 664                 *cs++ = i915_mmio_reg_offset(wa->reg);
 665                 *cs++ = wa->val;
 666         }
 667         *cs++ = MI_NOOP;
 668
 669         intel_ring_advance(rq, cs);
 670
 671         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 672         if (ret)
 673                 return ret;
 674
 675         return 0;
 676 }
 677
 678 static void
 679 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 680 {
 681         /* WaDisableKillLogic:bxt,skl,kbl */
 682         if (!IS_COFFEELAKE(i915))
 683                 wa_write_or(wal,
 684                             GAM_ECOCHK,
 685                             ECOCHK_DIS_TLB);
 686
 687         if (HAS_LLC(i915)) {
 688                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 689                  *
 690                  * Must match Display Engine. See
 691                  * WaCompressedResourceDisplayNewHashMode.
 692                  */
 693                 wa_write_or(wal,
 694                             MMCD_MISC_CTRL,
 695                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 696         }
 697
 698         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 699         wa_write_or(wal,
 700                     GAM_ECOCHK,
 701                     BDW_DISABLE_HDC_INVALIDATION);
 702 }
 703
 704 static void
 705 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 706 {
 707         gen9_gt_workarounds_init(i915, wal);
 708
 709         /* WaDisableGafsUnitClkGating:skl */
 710         wa_write_or(wal,
 711                     GEN7_UCGCTL4,
 712                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 713
 714         /* WaInPlaceDecompressionHang:skl */
 715         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 716                 wa_write_or(wal,
 717                             GEN9_GAMT_ECO_REG_RW_IA,
 718                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 719 }
 720
 721 static void
 722 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 723 {
 724         gen9_gt_workarounds_init(i915, wal);
 725
 726         /* WaInPlaceDecompressionHang:bxt */
 727         wa_write_or(wal,
 728                     GEN9_GAMT_ECO_REG_RW_IA,
 729                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 730 }
 731
 732 static void
 733 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 734 {
 735         gen9_gt_workarounds_init(i915, wal);
 736
 737         /* WaDisableDynamicCreditSharing:kbl */
 738         if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
 739                 wa_write_or(wal,
 740                             GAMT_CHKN_BIT_REG,
 741                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 742
 743         /* WaDisableGafsUnitClkGating:kbl */
 744         wa_write_or(wal,
 745                     GEN7_UCGCTL4,
 746                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 747
 748         /* WaInPlaceDecompressionHang:kbl */
 749         wa_write_or(wal,
 750                     GEN9_GAMT_ECO_REG_RW_IA,
 751                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 752 }
 753
 754 static void
 755 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 756 {
 757         gen9_gt_workarounds_init(i915, wal);
 758 }
 759
 760 static void
 761 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 762 {
 763         gen9_gt_workarounds_init(i915, wal);
 764
 765         /* WaDisableGafsUnitClkGating:cfl */
 766         wa_write_or(wal,
 767                     GEN7_UCGCTL4,
 768                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 769
 770         /* WaInPlaceDecompressionHang:cfl */
 771         wa_write_or(wal,
 772                     GEN9_GAMT_ECO_REG_RW_IA,
 773                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 774 }
 775
 776 static void
 777 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 778 {
 779         const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
 780         unsigned int slice, subslice;
 781         u32 l3_en, mcr, mcr_mask;
 782
 783         GEM_BUG_ON(INTEL_GEN(i915) < 10);
 784
 785         /*
 786          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
 787          * L3Banks could be fused off in single slice scenario. If that is
 788          * the case, we might need to program MCR select to a valid L3Bank
 789          * by default, to make sure we correctly read certain registers
 790          * later on (in the range 0xB100 - 0xB3FF).
 791          *
 792          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
 793          * Before any MMIO read into slice/subslice specific registers, MCR
 794          * packet control register needs to be programmed to point to any
 795          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 796          * This means each subsequent MMIO read will be forwarded to an
 797          * specific s/ss combination, but this is OK since these registers
 798          * are consistent across s/ss in almost all cases. In the rare
 799          * occasions, such as INSTDONE, where this value is dependent
 800          * on s/ss combo, the read should be done with read_subslice_reg.
 801          *
 802          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
 803          * to which subslice, or to which L3 bank, the respective mmio reads
 804          * will go, we have to find a common index which works for both
 805          * accesses.
 806          *
 807          * Case where we cannot find a common index fortunately should not
 808          * happen in production hardware, so we only emit a warning instead of
 809          * implementing something more complex that requires checking the range
 810          * of every MMIO read.
 811          */
 812
 813         if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
 814                 u32 l3_fuse =
 815                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
 816                         GEN10_L3BANK_MASK;
 817
 818                 DRM_DEBUG_DRIVER("L3 fuse = %x\n", l3_fuse);
 819                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
 820         } else {
 821                 l3_en = ~0;
 822         }
 823
 824         slice = fls(sseu->slice_mask) - 1;
 825         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
 826         if (!subslice) {
 827                 DRM_WARN("No common index found between subslice mask %x and L3 bank mask %x!\n",
 828                          intel_sseu_get_subslices(sseu, slice), l3_en);
 829                 subslice = fls(l3_en);
 830                 WARN_ON(!subslice);
 831         }
 832         subslice--;
 833
 834         if (INTEL_GEN(i915) >= 11) {
 835                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
 836                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
 837         } else {
 838                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
 839                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
 840         }
 841
 842         DRM_DEBUG_DRIVER("MCR slice/subslice = %x\n", mcr);
 843
 844         wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
 845 }
 846
 847 static void
 848 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 849 {
 850         wa_init_mcr(i915, wal);
 851
 852         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
 853         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 854                 wa_write_or(wal,
 855                             GAMT_CHKN_BIT_REG,
 856                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
 857
 858         /* WaInPlaceDecompressionHang:cnl */
 859         wa_write_or(wal,
 860                     GEN9_GAMT_ECO_REG_RW_IA,
 861                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 862 }
 863
 864 static void
 865 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 866 {
 867         wa_init_mcr(i915, wal);
 868
 869         /* WaInPlaceDecompressionHang:icl */
 870         wa_write_or(wal,
 871                     GEN9_GAMT_ECO_REG_RW_IA,
 872                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 873
 874         /* WaModifyGamTlbPartitioning:icl */
 875         wa_write_masked_or(wal,
 876                            GEN11_GACB_PERF_CTRL,
 877                            GEN11_HASH_CTRL_MASK,
 878                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
 879
 880         /* Wa_1405766107:icl
 881          * Formerly known as WaCL2SFHalfMaxAlloc
 882          */
 883         wa_write_or(wal,
 884                     GEN11_LSN_UNSLCVC,
 885                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
 886                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
 887
 888         /* Wa_220166154:icl
 889          * Formerly known as WaDisCtxReload
 890          */
 891         wa_write_or(wal,
 892                     GEN8_GAMW_ECO_DEV_RW_IA,
 893                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
 894
 895         /* Wa_1405779004:icl (pre-prod) */
 896         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 897                 wa_write_or(wal,
 898                             SLICE_UNIT_LEVEL_CLKGATE,
 899                             MSCUNIT_CLKGATE_DIS);
 900
 901         /* Wa_1406680159:icl */
 902         wa_write_or(wal,
 903                     SUBSLICE_UNIT_LEVEL_CLKGATE,
 904                     GWUNIT_CLKGATE_DIS);
 905
 906         /* Wa_1406838659:icl (pre-prod) */
 907         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 908                 wa_write_or(wal,
 909                             INF_UNIT_LEVEL_CLKGATE,
 910                             CGPSF_CLKGATE_DIS);
 911
 912         /* Wa_1406463099:icl
 913          * Formerly known as WaGamTlbPendError
 914          */
 915         wa_write_or(wal,
 916                     GAMT_CHKN_BIT_REG,
 917                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
 918
 919         /* Wa_1607087056:icl */
 920         wa_write_or(wal,
 921                     SLICE_UNIT_LEVEL_CLKGATE,
 922                     L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
 923 }
 924
 925 static void
 926 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 927 {
 928         /* Wa_1409420604:tgl */
 929         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
 930                 wa_write_or(wal,
 931                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
 932                             CPSSUNIT_CLKGATE_DIS);
 933
 934         /* Wa_1409180338:tgl */
 935         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
 936                 wa_write_or(wal,
 937                             SLICE_UNIT_LEVEL_CLKGATE,
 938                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
 939 }
 940
 941 static void
 942 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 943 {
 944         if (IS_GEN(i915, 12))
 945                 tgl_gt_workarounds_init(i915, wal);
 946         else if (IS_GEN(i915, 11))
 947                 icl_gt_workarounds_init(i915, wal);
 948         else if (IS_CANNONLAKE(i915))
 949                 cnl_gt_workarounds_init(i915, wal);
 950         else if (IS_COFFEELAKE(i915))
 951                 cfl_gt_workarounds_init(i915, wal);
 952         else if (IS_GEMINILAKE(i915))
 953                 glk_gt_workarounds_init(i915, wal);
 954         else if (IS_KABYLAKE(i915))
 955                 kbl_gt_workarounds_init(i915, wal);
 956         else if (IS_BROXTON(i915))
 957                 bxt_gt_workarounds_init(i915, wal);
 958         else if (IS_SKYLAKE(i915))
 959                 skl_gt_workarounds_init(i915, wal);
 960         else if (INTEL_GEN(i915) <= 8)
 961                 return;
 962         else
 963                 MISSING_CASE(INTEL_GEN(i915));
 964 }
 965
 966 void intel_gt_init_workarounds(struct drm_i915_private *i915)
 967 {
 968         struct i915_wa_list *wal = &i915->gt_wa_list;
 969
 970         wa_init_start(wal, "GT", "global");
 971         gt_init_workarounds(i915, wal);
 972         wa_init_finish(wal);
 973 }
 974
 975 static enum forcewake_domains
 976 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 977 {
 978         enum forcewake_domains fw = 0;
 979         struct i915_wa *wa;
 980         unsigned int i;
 981
 982         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 983                 fw |= intel_uncore_forcewake_for_reg(uncore,
 984                                                      wa->reg,
 985                                                      FW_REG_READ |
 986                                                      FW_REG_WRITE);
 987
 988         return fw;
 989 }
 990
 991 static bool
 992 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
 993 {
 994         if ((cur ^ wa->val) & wa->read) {
 995                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x, mask=%x)\n",
 996                           name, from, i915_mmio_reg_offset(wa->reg),
 997                           cur, cur & wa->read,
 998                           wa->val, wa->mask);
 999
1000                 return false;
1001         }
1002
1003         return true;
1004 }
1005
1006 static void
1007 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1008 {
1009         enum forcewake_domains fw;
1010         unsigned long flags;
1011         struct i915_wa *wa;
1012         unsigned int i;
1013
1014         if (!wal->count)
1015                 return;
1016
1017         fw = wal_get_fw_for_rmw(uncore, wal);
1018
1019         spin_lock_irqsave(&uncore->lock, flags);
1020         intel_uncore_forcewake_get__locked(uncore, fw);
1021
1022         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1023                 intel_uncore_rmw_fw(uncore, wa->reg, wa->mask, wa->val);
1024                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1025                         wa_verify(wa,
1026                                   intel_uncore_read_fw(uncore, wa->reg),
1027                                   wal->name, "application");
1028         }
1029
1030         intel_uncore_forcewake_put__locked(uncore, fw);
1031         spin_unlock_irqrestore(&uncore->lock, flags);
1032 }
1033
1034 void intel_gt_apply_workarounds(struct intel_gt *gt)
1035 {
1036         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1037 }
1038
1039 static bool wa_list_verify(struct intel_uncore *uncore,
1040                            const struct i915_wa_list *wal,
1041                            const char *from)
1042 {
1043         struct i915_wa *wa;
1044         unsigned int i;
1045         bool ok = true;
1046
1047         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1048                 ok &= wa_verify(wa,
1049                                 intel_uncore_read(uncore, wa->reg),
1050                                 wal->name, from);
1051
1052         return ok;
1053 }
1054
1055 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1056 {
1057         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1058 }
1059
1060 static inline bool is_nonpriv_flags_valid(u32 flags)
1061 {
1062         /* Check only valid flag bits are set */
1063         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1064                 return false;
1065
1066         /* NB: Only 3 out of 4 enum values are valid for access field */
1067         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1068             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1069                 return false;
1070
1071         return true;
1072 }
1073
1074 static void
1075 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1076 {
1077         struct i915_wa wa = {
1078                 .reg = reg
1079         };
1080
1081         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1082                 return;
1083
1084         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1085                 return;
1086
1087         wa.reg.reg |= flags;
1088         _wa_add(wal, &wa);
1089 }
1090
1091 static void
1092 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1093 {
1094         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1095 }
1096
1097 static void gen9_whitelist_build(struct i915_wa_list *w)
1098 {
1099         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1100         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1101
1102         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1103         whitelist_reg(w, GEN8_CS_CHICKEN1);
1104
1105         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1106         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1107
1108         /* WaSendPushConstantsFromMMIO:skl,bxt */
1109         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1110 }
1111
1112 static void skl_whitelist_build(struct intel_engine_cs *engine)
1113 {
1114         struct i915_wa_list *w = &engine->whitelist;
1115
1116         if (engine->class != RENDER_CLASS)
1117                 return;
1118
1119         gen9_whitelist_build(w);
1120
1121         /* WaDisableLSQCROPERFforOCL:skl */
1122         whitelist_reg(w, GEN8_L3SQCREG4);
1123 }
1124
1125 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1126 {
1127         if (engine->class != RENDER_CLASS)
1128                 return;
1129
1130         gen9_whitelist_build(&engine->whitelist);
1131 }
1132
1133 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1134 {
1135         struct i915_wa_list *w = &engine->whitelist;
1136
1137         if (engine->class != RENDER_CLASS)
1138                 return;
1139
1140         gen9_whitelist_build(w);
1141
1142         /* WaDisableLSQCROPERFforOCL:kbl */
1143         whitelist_reg(w, GEN8_L3SQCREG4);
1144 }
1145
1146 static void glk_whitelist_build(struct intel_engine_cs *engine)
1147 {
1148         struct i915_wa_list *w = &engine->whitelist;
1149
1150         if (engine->class != RENDER_CLASS)
1151                 return;
1152
1153         gen9_whitelist_build(w);
1154
1155         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1156         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1157 }
1158
1159 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1160 {
1161         struct i915_wa_list *w = &engine->whitelist;
1162
1163         if (engine->class != RENDER_CLASS)
1164                 return;
1165
1166         gen9_whitelist_build(w);
1167
1168         /*
1169          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1170          *
1171          * This covers 4 register which are next to one another :
1172          *   - PS_INVOCATION_COUNT
1173          *   - PS_INVOCATION_COUNT_UDW
1174          *   - PS_DEPTH_COUNT
1175          *   - PS_DEPTH_COUNT_UDW
1176          */
1177         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1178                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1179                           RING_FORCE_TO_NONPRIV_RANGE_4);
1180 }
1181
1182 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1183 {
1184         struct i915_wa_list *w = &engine->whitelist;
1185
1186         if (engine->class != RENDER_CLASS)
1187                 return;
1188
1189         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1190         whitelist_reg(w, GEN8_CS_CHICKEN1);
1191 }
1192
1193 static void icl_whitelist_build(struct intel_engine_cs *engine)
1194 {
1195         struct i915_wa_list *w = &engine->whitelist;
1196
1197         switch (engine->class) {
1198         case RENDER_CLASS:
1199                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1200                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1201
1202                 /* WaAllowUMDToModifySamplerMode:icl */
1203                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1204
1205                 /* WaEnableStateCacheRedirectToCS:icl */
1206                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1207
1208                 /*
1209                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1210                  *
1211                  * This covers 4 register which are next to one another :
1212                  *   - PS_INVOCATION_COUNT
1213                  *   - PS_INVOCATION_COUNT_UDW
1214                  *   - PS_DEPTH_COUNT
1215                  *   - PS_DEPTH_COUNT_UDW
1216                  */
1217                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1218                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1219                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1220                 break;
1221
1222         case VIDEO_DECODE_CLASS:
1223                 /* hucStatusRegOffset */
1224                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1225                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1226                 /* hucUKernelHdrInfoRegOffset */
1227                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1228                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1229                 /* hucStatus2RegOffset */
1230                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1231                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1232                 break;
1233
1234         default:
1235                 break;
1236         }
1237 }
1238
1239 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1240 {
1241         struct i915_wa_list *w = &engine->whitelist;
1242
1243         switch (engine->class) {
1244         case RENDER_CLASS:
1245                 /*
1246                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1247                  *
1248                  * This covers 4 registers which are next to one another :
1249                  *   - PS_INVOCATION_COUNT
1250                  *   - PS_INVOCATION_COUNT_UDW
1251                  *   - PS_DEPTH_COUNT
1252                  *   - PS_DEPTH_COUNT_UDW
1253                  */
1254                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1255                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1256                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1257                 break;
1258         default:
1259                 break;
1260         }
1261 }
1262
1263 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1264 {
1265         struct drm_i915_private *i915 = engine->i915;
1266         struct i915_wa_list *w = &engine->whitelist;
1267
1268         wa_init_start(w, "whitelist", engine->name);
1269
1270         if (IS_GEN(i915, 12))
1271                 tgl_whitelist_build(engine);
1272         else if (IS_GEN(i915, 11))
1273                 icl_whitelist_build(engine);
1274         else if (IS_CANNONLAKE(i915))
1275                 cnl_whitelist_build(engine);
1276         else if (IS_COFFEELAKE(i915))
1277                 cfl_whitelist_build(engine);
1278         else if (IS_GEMINILAKE(i915))
1279                 glk_whitelist_build(engine);
1280         else if (IS_KABYLAKE(i915))
1281                 kbl_whitelist_build(engine);
1282         else if (IS_BROXTON(i915))
1283                 bxt_whitelist_build(engine);
1284         else if (IS_SKYLAKE(i915))
1285                 skl_whitelist_build(engine);
1286         else if (INTEL_GEN(i915) <= 8)
1287                 return;
1288         else
1289                 MISSING_CASE(INTEL_GEN(i915));
1290
1291         wa_init_finish(w);
1292 }
1293
1294 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1295 {
1296         const struct i915_wa_list *wal = &engine->whitelist;
1297         struct intel_uncore *uncore = engine->uncore;
1298         const u32 base = engine->mmio_base;
1299         struct i915_wa *wa;
1300         unsigned int i;
1301
1302         if (!wal->count)
1303                 return;
1304
1305         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1306                 intel_uncore_write(uncore,
1307                                    RING_FORCE_TO_NONPRIV(base, i),
1308                                    i915_mmio_reg_offset(wa->reg));
1309
1310         /* And clear the rest just in case of garbage */
1311         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1312                 intel_uncore_write(uncore,
1313                                    RING_FORCE_TO_NONPRIV(base, i),
1314                                    i915_mmio_reg_offset(RING_NOPID(base)));
1315 }
1316
1317 static void
1318 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1319 {
1320         struct drm_i915_private *i915 = engine->i915;
1321
1322         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1323                 /* Wa_1606700617:tgl */
1324                 wa_masked_en(wal,
1325                              GEN9_CS_DEBUG_MODE1,
1326                              FF_DOP_CLOCK_GATE_DISABLE);
1327
1328                 /* Wa_1607138336:tgl */
1329                 wa_write_or(wal,
1330                             GEN9_CTX_PREEMPT_REG,
1331                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1332
1333                 /* Wa_1607030317:tgl */
1334                 /* Wa_1607186500:tgl */
1335                 /* Wa_1607297627:tgl */
1336                 wa_masked_en(wal,
1337                              GEN6_RC_SLEEP_PSMI_CONTROL,
1338                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1339                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1340
1341                 /*
1342                  * Wa_1606679103:tgl
1343                  * (see also Wa_1606682166:icl)
1344                  */
1345                 wa_write_or(wal,
1346                             GEN7_SARCHKMD,
1347                             GEN7_DISABLE_SAMPLER_PREFETCH);
1348         }
1349
1350         if (IS_GEN(i915, 11)) {
1351                 /* This is not an Wa. Enable for better image quality */
1352                 wa_masked_en(wal,
1353                              _3D_CHICKEN3,
1354                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1355
1356                 /* WaPipelineFlushCoherentLines:icl */
1357                 wa_write_or(wal,
1358                             GEN8_L3SQCREG4,
1359                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1360
1361                 /*
1362                  * Wa_1405543622:icl
1363                  * Formerly known as WaGAPZPriorityScheme
1364                  */
1365                 wa_write_or(wal,
1366                             GEN8_GARBCNTL,
1367                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1368
1369                 /*
1370                  * Wa_1604223664:icl
1371                  * Formerly known as WaL3BankAddressHashing
1372                  */
1373                 wa_write_masked_or(wal,
1374                                    GEN8_GARBCNTL,
1375                                    GEN11_HASH_CTRL_EXCL_MASK,
1376                                    GEN11_HASH_CTRL_EXCL_BIT0);
1377                 wa_write_masked_or(wal,
1378                                    GEN11_GLBLINVL,
1379                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1380                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1381
1382                 /*
1383                  * Wa_1405733216:icl
1384                  * Formerly known as WaDisableCleanEvicts
1385                  */
1386                 wa_write_or(wal,
1387                             GEN8_L3SQCREG4,
1388                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1389
1390                 /* WaForwardProgressSoftReset:icl */
1391                 wa_write_or(wal,
1392                             GEN10_SCRATCH_LNCF2,
1393                             PMFLUSHDONE_LNICRSDROP |
1394                             PMFLUSH_GAPL3UNBLOCK |
1395                             PMFLUSHDONE_LNEBLK);
1396
1397                 /* Wa_1406609255:icl (pre-prod) */
1398                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1399                         wa_write_or(wal,
1400                                     GEN7_SARCHKMD,
1401                                     GEN7_DISABLE_DEMAND_PREFETCH);
1402
1403                 /* Wa_1606682166:icl */
1404                 wa_write_or(wal,
1405                             GEN7_SARCHKMD,
1406                             GEN7_DISABLE_SAMPLER_PREFETCH);
1407
1408                 /* Wa_1409178092:icl */
1409                 wa_write_masked_or(wal,
1410                                    GEN11_SCRATCH2,
1411                                    GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1412                                    0);
1413         }
1414
1415         if (IS_GEN_RANGE(i915, 9, 11)) {
1416                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */
1417                 wa_masked_en(wal,
1418                              GEN7_FF_SLICE_CS_CHICKEN1,
1419                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1420         }
1421
1422         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1423                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1424                 wa_write_or(wal,
1425                             GEN8_GARBCNTL,
1426                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1427         }
1428
1429         if (IS_BROXTON(i915)) {
1430                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1431                 wa_masked_en(wal,
1432                              FF_SLICE_CS_CHICKEN2,
1433                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1434         }
1435
1436         if (IS_GEN(i915, 9)) {
1437                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1438                 wa_masked_en(wal,
1439                              GEN9_CSFE_CHICKEN1_RCS,
1440                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1441
1442                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1443                 wa_write_or(wal,
1444                             BDW_SCRATCH1,
1445                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1446
1447                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1448                 if (IS_GEN9_LP(i915))
1449                         wa_write_masked_or(wal,
1450                                            GEN8_L3SQCREG1,
1451                                            L3_PRIO_CREDITS_MASK,
1452                                            L3_GENERAL_PRIO_CREDITS(62) |
1453                                            L3_HIGH_PRIO_CREDITS(2));
1454
1455                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1456                 wa_write_or(wal,
1457                             GEN8_L3SQCREG4,
1458                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1459         }
1460 }
1461
1462 static void
1463 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1464 {
1465         struct drm_i915_private *i915 = engine->i915;
1466
1467         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1468         if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1469                 wa_write(wal,
1470                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1471                          1);
1472         }
1473 }
1474
1475 static void
1476 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1477 {
1478         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8))
1479                 return;
1480
1481         if (engine->class == RENDER_CLASS)
1482                 rcs_engine_wa_init(engine, wal);
1483         else
1484                 xcs_engine_wa_init(engine, wal);
1485 }
1486
1487 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1488 {
1489         struct i915_wa_list *wal = &engine->wa_list;
1490
1491         if (INTEL_GEN(engine->i915) < 8)
1492                 return;
1493
1494         wa_init_start(wal, "engine", engine->name);
1495         engine_init_workarounds(engine, wal);
1496         wa_init_finish(wal);
1497 }
1498
1499 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1500 {
1501         wa_list_apply(engine->uncore, &engine->wa_list);
1502 }
1503
1504 static struct i915_vma *
1505 create_scratch(struct i915_address_space *vm, int count)
1506 {
1507         struct drm_i915_gem_object *obj;
1508         struct i915_vma *vma;
1509         unsigned int size;
1510         int err;
1511
1512         size = round_up(count * sizeof(u32), PAGE_SIZE);
1513         obj = i915_gem_object_create_internal(vm->i915, size);
1514         if (IS_ERR(obj))
1515                 return ERR_CAST(obj);
1516
1517         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1518
1519         vma = i915_vma_instance(obj, vm, NULL);
1520         if (IS_ERR(vma)) {
1521                 err = PTR_ERR(vma);
1522                 goto err_obj;
1523         }
1524
1525         err = i915_vma_pin(vma, 0, 0,
1526                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1527         if (err)
1528                 goto err_obj;
1529
1530         return vma;
1531
1532 err_obj:
1533         i915_gem_object_put(obj);
1534         return ERR_PTR(err);
1535 }
1536
1537 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
1538 {
1539         /*
1540          * Registers in this range are affected by the MCR selector
1541          * which only controls CPU initiated MMIO. Routing does not
1542          * work for CS access so we cannot verify them on this path.
1543          */
1544         if (INTEL_GEN(i915) >= 8 && (offset >= 0xb000 && offset <= 0xb4ff))
1545                 return true;
1546
1547         return false;
1548 }
1549
1550 static int
1551 wa_list_srm(struct i915_request *rq,
1552             const struct i915_wa_list *wal,
1553             struct i915_vma *vma)
1554 {
1555         struct drm_i915_private *i915 = rq->i915;
1556         unsigned int i, count = 0;
1557         const struct i915_wa *wa;
1558         u32 srm, *cs;
1559
1560         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1561         if (INTEL_GEN(i915) >= 8)
1562                 srm++;
1563
1564         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1565                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
1566                         count++;
1567         }
1568
1569         cs = intel_ring_begin(rq, 4 * count);
1570         if (IS_ERR(cs))
1571                 return PTR_ERR(cs);
1572
1573         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1574                 u32 offset = i915_mmio_reg_offset(wa->reg);
1575
1576                 if (mcr_range(i915, offset))
1577                         continue;
1578
1579                 *cs++ = srm;
1580                 *cs++ = offset;
1581                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1582                 *cs++ = 0;
1583         }
1584         intel_ring_advance(rq, cs);
1585
1586         return 0;
1587 }
1588
1589 static int engine_wa_list_verify(struct intel_context *ce,
1590                                  const struct i915_wa_list * const wal,
1591                                  const char *from)
1592 {
1593         const struct i915_wa *wa;
1594         struct i915_request *rq;
1595         struct i915_vma *vma;
1596         unsigned int i;
1597         u32 *results;
1598         int err;
1599
1600         if (!wal->count)
1601                 return 0;
1602
1603         vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
1604         if (IS_ERR(vma))
1605                 return PTR_ERR(vma);
1606
1607         intel_engine_pm_get(ce->engine);
1608         rq = intel_context_create_request(ce);
1609         intel_engine_pm_put(ce->engine);
1610         if (IS_ERR(rq)) {
1611                 err = PTR_ERR(rq);
1612                 goto err_vma;
1613         }
1614
1615         err = wa_list_srm(rq, wal, vma);
1616         if (err)
1617                 goto err_vma;
1618
1619         i915_request_get(rq);
1620         i915_request_add(rq);
1621         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1622                 err = -ETIME;
1623                 goto err_rq;
1624         }
1625
1626         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1627         if (IS_ERR(results)) {
1628                 err = PTR_ERR(results);
1629                 goto err_rq;
1630         }
1631
1632         err = 0;
1633         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1634                 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg)))
1635                         continue;
1636
1637                 if (!wa_verify(wa, results[i], wal->name, from))
1638                         err = -ENXIO;
1639         }
1640
1641         i915_gem_object_unpin_map(vma->obj);
1642
1643 err_rq:
1644         i915_request_put(rq);
1645 err_vma:
1646         i915_vma_unpin(vma);
1647         i915_vma_put(vma);
1648         return err;
1649 }
1650
1651 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
1652                                     const char *from)
1653 {
1654         return engine_wa_list_verify(engine->kernel_context,
1655                                      &engine->wa_list,
1656                                      from);
1657 }
1658
1659 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1660 #include "selftest_workarounds.c"
1661 #endif