drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26 #include "i915_vgpu.h"
  27
  28 #include <linux/pm_runtime.h>
  29
  30 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  31
  32 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
  33
  34 static const char * const forcewake_domain_names[] = {
  35         "render",
  36         "blitter",
  37         "media",
  38 };
  39
  40 const char *
  41 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
  42 {
  43         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
  44
  45         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
  46                 return forcewake_domain_names[id];
  47
  48         WARN_ON(id);
  49
  50         return "unknown";
  51 }
  52
  53 static inline void
  54 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
  55 {
  56         WARN_ON(!i915_mmio_reg_valid(d->reg_set));
  57         __raw_i915_write32(d->i915, d->reg_set, d->val_reset);
  58 }
  59
  60 static inline void
  61 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
  62 {
  63         d->wake_count++;
  64         hrtimer_start_range_ns(&d->timer,
  65                                NSEC_PER_MSEC,
  66                                NSEC_PER_MSEC,
  67                                HRTIMER_MODE_REL);
  68 }
  69
  70 static inline void
  71 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
  72 {
  73         if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
  74                              FORCEWAKE_KERNEL) == 0,
  75                             FORCEWAKE_ACK_TIMEOUT_MS))
  76                 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
  77                           intel_uncore_forcewake_domain_to_str(d->id));
  78 }
  79
  80 static inline void
  81 fw_domain_get(const struct intel_uncore_forcewake_domain *d)
  82 {
  83         __raw_i915_write32(d->i915, d->reg_set, d->val_set);
  84 }
  85
  86 static inline void
  87 fw_domain_wait_ack(const struct intel_uncore_forcewake_domain *d)
  88 {
  89         if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
  90                              FORCEWAKE_KERNEL),
  91                             FORCEWAKE_ACK_TIMEOUT_MS))
  92                 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
  93                           intel_uncore_forcewake_domain_to_str(d->id));
  94 }
  95
  96 static inline void
  97 fw_domain_put(const struct intel_uncore_forcewake_domain *d)
  98 {
  99         __raw_i915_write32(d->i915, d->reg_set, d->val_clear);
 100 }
 101
 102 static inline void
 103 fw_domain_posting_read(const struct intel_uncore_forcewake_domain *d)
 104 {
 105         /* something from same cacheline, but not from the set register */
 106         if (i915_mmio_reg_valid(d->reg_post))
 107                 __raw_posting_read(d->i915, d->reg_post);
 108 }
 109
 110 static void
 111 fw_domains_get(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
 112 {
 113         struct intel_uncore_forcewake_domain *d;
 114
 115         for_each_fw_domain_masked(d, fw_domains, dev_priv) {
 116                 fw_domain_wait_ack_clear(d);
 117                 fw_domain_get(d);
 118         }
 119
 120         for_each_fw_domain_masked(d, fw_domains, dev_priv)
 121                 fw_domain_wait_ack(d);
 122 }
 123
 124 static void
 125 fw_domains_put(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
 126 {
 127         struct intel_uncore_forcewake_domain *d;
 128
 129         for_each_fw_domain_masked(d, fw_domains, dev_priv) {
 130                 fw_domain_put(d);
 131                 fw_domain_posting_read(d);
 132         }
 133 }
 134
 135 static void
 136 vgpu_fw_domains_nop(struct drm_i915_private *dev_priv,
 137                     enum forcewake_domains fw_domains)
 138 {
 139         /* Guest driver doesn't need to takes care forcewake. */
 140 }
 141
 142 static void
 143 fw_domains_posting_read(struct drm_i915_private *dev_priv)
 144 {
 145         struct intel_uncore_forcewake_domain *d;
 146
 147         /* No need to do for all, just do for first found */
 148         for_each_fw_domain(d, dev_priv) {
 149                 fw_domain_posting_read(d);
 150                 break;
 151         }
 152 }
 153
 154 static void
 155 fw_domains_reset(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
 156 {
 157         struct intel_uncore_forcewake_domain *d;
 158
 159         if (dev_priv->uncore.fw_domains == 0)
 160                 return;
 161
 162         for_each_fw_domain_masked(d, fw_domains, dev_priv)
 163                 fw_domain_reset(d);
 164
 165         fw_domains_posting_read(dev_priv);
 166 }
 167
 168 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
 169 {
 170         /* w/a for a sporadic read returning 0 by waiting for the GT
 171          * thread to wake up.
 172          */
 173         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
 174                                 GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
 175                 DRM_ERROR("GT thread status wait timed out\n");
 176 }
 177
 178 static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
 179                                               enum forcewake_domains fw_domains)
 180 {
 181         fw_domains_get(dev_priv, fw_domains);
 182
 183         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 184         __gen6_gt_wait_for_thread_c0(dev_priv);
 185 }
 186
 187 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 188 {
 189         u32 gtfifodbg;
 190
 191         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 192         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 193                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 194 }
 195
 196 static void fw_domains_put_with_fifo(struct drm_i915_private *dev_priv,
 197                                      enum forcewake_domains fw_domains)
 198 {
 199         fw_domains_put(dev_priv, fw_domains);
 200         gen6_gt_check_fifodbg(dev_priv);
 201 }
 202
 203 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
 204 {
 205         u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
 206
 207         return count & GT_FIFO_FREE_ENTRIES_MASK;
 208 }
 209
 210 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 211 {
 212         int ret = 0;
 213
 214         /* On VLV, FIFO will be shared by both SW and HW.
 215          * So, we need to read the FREE_ENTRIES everytime */
 216         if (IS_VALLEYVIEW(dev_priv))
 217                 dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv);
 218
 219         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 220                 int loop = 500;
 221                 u32 fifo = fifo_free_entries(dev_priv);
 222
 223                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 224                         udelay(10);
 225                         fifo = fifo_free_entries(dev_priv);
 226                 }
 227                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 228                         ++ret;
 229                 dev_priv->uncore.fifo_count = fifo;
 230         }
 231         dev_priv->uncore.fifo_count--;
 232
 233         return ret;
 234 }
 235
 236 static enum hrtimer_restart
 237 intel_uncore_fw_release_timer(struct hrtimer *timer)
 238 {
 239         struct intel_uncore_forcewake_domain *domain =
 240                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 241         struct drm_i915_private *dev_priv = domain->i915;
 242         unsigned long irqflags;
 243
 244         assert_rpm_device_not_suspended(dev_priv);
 245
 246         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 247         if (WARN_ON(domain->wake_count == 0))
 248                 domain->wake_count++;
 249
 250         if (--domain->wake_count == 0) {
 251                 dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
 252                 dev_priv->uncore.fw_domains_active &= ~domain->mask;
 253         }
 254
 255         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 256
 257         return HRTIMER_NORESTART;
 258 }
 259
 260 void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 261                                   bool restore)
 262 {
 263         unsigned long irqflags;
 264         struct intel_uncore_forcewake_domain *domain;
 265         int retry_count = 100;
 266         enum forcewake_domains fw, active_domains;
 267
 268         /* Hold uncore.lock across reset to prevent any register access
 269          * with forcewake not set correctly. Wait until all pending
 270          * timers are run before holding.
 271          */
 272         while (1) {
 273                 active_domains = 0;
 274
 275                 for_each_fw_domain(domain, dev_priv) {
 276                         if (hrtimer_cancel(&domain->timer) == 0)
 277                                 continue;
 278
 279                         intel_uncore_fw_release_timer(&domain->timer);
 280                 }
 281
 282                 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 283
 284                 for_each_fw_domain(domain, dev_priv) {
 285                         if (hrtimer_active(&domain->timer))
 286                                 active_domains |= domain->mask;
 287                 }
 288
 289                 if (active_domains == 0)
 290                         break;
 291
 292                 if (--retry_count == 0) {
 293                         DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
 294                         break;
 295                 }
 296
 297                 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 298                 cond_resched();
 299         }
 300
 301         WARN_ON(active_domains);
 302
 303         fw = dev_priv->uncore.fw_domains_active;
 304         if (fw)
 305                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 306
 307         fw_domains_reset(dev_priv, FORCEWAKE_ALL);
 308
 309         if (restore) { /* If reset with a user forcewake, try to restore */
 310                 if (fw)
 311                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 312
 313                 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
 314                         dev_priv->uncore.fifo_count =
 315                                 fifo_free_entries(dev_priv);
 316         }
 317
 318         if (!restore)
 319                 assert_forcewakes_inactive(dev_priv);
 320
 321         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 322 }
 323
 324 static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
 325 {
 326         const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
 327         const unsigned int sets[4] = { 1, 1, 2, 2 };
 328         const u32 cap = dev_priv->edram_cap;
 329
 330         return EDRAM_NUM_BANKS(cap) *
 331                 ways[EDRAM_WAYS_IDX(cap)] *
 332                 sets[EDRAM_SETS_IDX(cap)] *
 333                 1024 * 1024;
 334 }
 335
 336 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
 337 {
 338         if (!HAS_EDRAM(dev_priv))
 339                 return 0;
 340
 341         /* The needed capability bits for size calculation
 342          * are not there with pre gen9 so return 128MB always.
 343          */
 344         if (INTEL_GEN(dev_priv) < 9)
 345                 return 128 * 1024 * 1024;
 346
 347         return gen9_edram_size(dev_priv);
 348 }
 349
 350 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
 351 {
 352         if (IS_HASWELL(dev_priv) ||
 353             IS_BROADWELL(dev_priv) ||
 354             INTEL_GEN(dev_priv) >= 9) {
 355                 dev_priv->edram_cap = __raw_i915_read32(dev_priv,
 356                                                         HSW_EDRAM_CAP);
 357
 358                 /* NB: We can't write IDICR yet because we do not have gt funcs
 359                  * set up */
 360         } else {
 361                 dev_priv->edram_cap = 0;
 362         }
 363
 364         if (HAS_EDRAM(dev_priv))
 365                 DRM_INFO("Found %lluMB of eDRAM\n",
 366                          intel_uncore_edram_size(dev_priv) / (1024 * 1024));
 367 }
 368
 369 static bool
 370 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 371 {
 372         u32 dbg;
 373
 374         dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
 375         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 376                 return false;
 377
 378         __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 379
 380         return true;
 381 }
 382
 383 static bool
 384 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 385 {
 386         u32 cer;
 387
 388         cer = __raw_i915_read32(dev_priv, CLAIM_ER);
 389         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 390                 return false;
 391
 392         __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
 393
 394         return true;
 395 }
 396
 397 static bool
 398 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 399 {
 400         if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
 401                 return fpga_check_for_unclaimed_mmio(dev_priv);
 402
 403         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 404                 return vlv_check_for_unclaimed_mmio(dev_priv);
 405
 406         return false;
 407 }
 408
 409 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 410                                           bool restore_forcewake)
 411 {
 412         struct intel_device_info *info = mkwrite_device_info(dev_priv);
 413
 414         /* clear out unclaimed reg detection bit */
 415         if (check_for_unclaimed_mmio(dev_priv))
 416                 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 417
 418         /* clear out old GT FIFO errors */
 419         if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
 420                 __raw_i915_write32(dev_priv, GTFIFODBG,
 421                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 422
 423         /* WaDisableShadowRegForCpd:chv */
 424         if (IS_CHERRYVIEW(dev_priv)) {
 425                 __raw_i915_write32(dev_priv, GTFIFOCTL,
 426                                    __raw_i915_read32(dev_priv, GTFIFOCTL) |
 427                                    GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 428                                    GT_FIFO_CTL_RC6_POLICY_STALL);
 429         }
 430
 431         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST))
 432                 info->has_decoupled_mmio = false;
 433
 434         intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
 435 }
 436
 437 void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 438                                  bool restore_forcewake)
 439 {
 440         __intel_uncore_early_sanitize(dev_priv, restore_forcewake);
 441         i915_check_and_clear_faults(dev_priv);
 442 }
 443
 444 void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 445 {
 446         i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
 447
 448         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 449         intel_sanitize_gt_powersave(dev_priv);
 450 }
 451
 452 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 453                                          enum forcewake_domains fw_domains)
 454 {
 455         struct intel_uncore_forcewake_domain *domain;
 456
 457         fw_domains &= dev_priv->uncore.fw_domains;
 458
 459         for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
 460                 if (domain->wake_count++)
 461                         fw_domains &= ~domain->mask;
 462         }
 463
 464         if (fw_domains) {
 465                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
 466                 dev_priv->uncore.fw_domains_active |= fw_domains;
 467         }
 468 }
 469
 470 /**
 471  * intel_uncore_forcewake_get - grab forcewake domain references
 472  * @dev_priv: i915 device instance
 473  * @fw_domains: forcewake domains to get reference on
 474  *
 475  * This function can be used get GT's forcewake domain references.
 476  * Normal register access will handle the forcewake domains automatically.
 477  * However if some sequence requires the GT to not power down a particular
 478  * forcewake domains this function should be called at the beginning of the
 479  * sequence. And subsequently the reference should be dropped by symmetric
 480  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 481  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 482  */
 483 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 484                                 enum forcewake_domains fw_domains)
 485 {
 486         unsigned long irqflags;
 487
 488         if (!dev_priv->uncore.funcs.force_wake_get)
 489                 return;
 490
 491         assert_rpm_wakelock_held(dev_priv);
 492
 493         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 494         __intel_uncore_forcewake_get(dev_priv, fw_domains);
 495         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 496 }
 497
 498 /**
 499  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 500  * @dev_priv: i915 device instance
 501  * @fw_domains: forcewake domains to get reference on
 502  *
 503  * See intel_uncore_forcewake_get(). This variant places the onus
 504  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 505  */
 506 void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
 507                                         enum forcewake_domains fw_domains)
 508 {
 509         lockdep_assert_held(&dev_priv->uncore.lock);
 510
 511         if (!dev_priv->uncore.funcs.force_wake_get)
 512                 return;
 513
 514         __intel_uncore_forcewake_get(dev_priv, fw_domains);
 515 }
 516
 517 static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 518                                          enum forcewake_domains fw_domains)
 519 {
 520         struct intel_uncore_forcewake_domain *domain;
 521
 522         fw_domains &= dev_priv->uncore.fw_domains;
 523
 524         for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
 525                 if (WARN_ON(domain->wake_count == 0))
 526                         continue;
 527
 528                 if (--domain->wake_count)
 529                         continue;
 530
 531                 fw_domain_arm_timer(domain);
 532         }
 533 }
 534
 535 /**
 536  * intel_uncore_forcewake_put - release a forcewake domain reference
 537  * @dev_priv: i915 device instance
 538  * @fw_domains: forcewake domains to put references
 539  *
 540  * This function drops the device-level forcewakes for specified
 541  * domains obtained by intel_uncore_forcewake_get().
 542  */
 543 void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 544                                 enum forcewake_domains fw_domains)
 545 {
 546         unsigned long irqflags;
 547
 548         if (!dev_priv->uncore.funcs.force_wake_put)
 549                 return;
 550
 551         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 552         __intel_uncore_forcewake_put(dev_priv, fw_domains);
 553         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 554 }
 555
 556 /**
 557  * intel_uncore_forcewake_put__locked - grab forcewake domain references
 558  * @dev_priv: i915 device instance
 559  * @fw_domains: forcewake domains to get reference on
 560  *
 561  * See intel_uncore_forcewake_put(). This variant places the onus
 562  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 563  */
 564 void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
 565                                         enum forcewake_domains fw_domains)
 566 {
 567         lockdep_assert_held(&dev_priv->uncore.lock);
 568
 569         if (!dev_priv->uncore.funcs.force_wake_put)
 570                 return;
 571
 572         __intel_uncore_forcewake_put(dev_priv, fw_domains);
 573 }
 574
 575 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 576 {
 577         if (!dev_priv->uncore.funcs.force_wake_get)
 578                 return;
 579
 580         WARN_ON(dev_priv->uncore.fw_domains_active);
 581 }
 582
 583 /* We give fast paths for the really cool registers */
 584 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
 585
 586 #define __gen6_reg_read_fw_domains(offset) \
 587 ({ \
 588         enum forcewake_domains __fwd; \
 589         if (NEEDS_FORCE_WAKE(offset)) \
 590                 __fwd = FORCEWAKE_RENDER; \
 591         else \
 592                 __fwd = 0; \
 593         __fwd; \
 594 })
 595
 596 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 597 {
 598         if (offset < entry->start)
 599                 return -1;
 600         else if (offset > entry->end)
 601                 return 1;
 602         else
 603                 return 0;
 604 }
 605
 606 /* Copied and "macroized" from lib/bsearch.c */
 607 #define BSEARCH(key, base, num, cmp) ({                                 \
 608         unsigned int start__ = 0, end__ = (num);                        \
 609         typeof(base) result__ = NULL;                                   \
 610         while (start__ < end__) {                                       \
 611                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 612                 int ret__ = (cmp)((key), (base) + mid__);               \
 613                 if (ret__ < 0) {                                        \
 614                         end__ = mid__;                                  \
 615                 } else if (ret__ > 0) {                                 \
 616                         start__ = mid__ + 1;                            \
 617                 } else {                                                \
 618                         result__ = (base) + mid__;                      \
 619                         break;                                          \
 620                 }                                                       \
 621         }                                                               \
 622         result__;                                                       \
 623 })
 624
 625 static enum forcewake_domains
 626 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
 627 {
 628         const struct intel_forcewake_range *entry;
 629
 630         entry = BSEARCH(offset,
 631                         dev_priv->uncore.fw_domains_table,
 632                         dev_priv->uncore.fw_domains_table_entries,
 633                         fw_range_cmp);
 634
 635         if (!entry)
 636                 return 0;
 637
 638         WARN(entry->domains & ~dev_priv->uncore.fw_domains,
 639              "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 640              entry->domains & ~dev_priv->uncore.fw_domains, offset);
 641
 642         return entry->domains;
 643 }
 644
 645 #define GEN_FW_RANGE(s, e, d) \
 646         { .start = (s), .end = (e), .domains = (d) }
 647
 648 #define HAS_FWTABLE(dev_priv) \
 649         (IS_GEN9(dev_priv) || \
 650          IS_CHERRYVIEW(dev_priv) || \
 651          IS_VALLEYVIEW(dev_priv))
 652
 653 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 654 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
 655         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 656         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
 657         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
 658         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 659         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
 660         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
 661         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 662 };
 663
 664 #define __fwtable_reg_read_fw_domains(offset) \
 665 ({ \
 666         enum forcewake_domains __fwd = 0; \
 667         if (NEEDS_FORCE_WAKE((offset))) \
 668                 __fwd = find_fw_domain(dev_priv, offset); \
 669         __fwd; \
 670 })
 671
 672 /* *Must* be sorted by offset! See intel_shadow_table_check(). */
 673 static const i915_reg_t gen8_shadowed_regs[] = {
 674         RING_TAIL(RENDER_RING_BASE),    /* 0x2000 (base) */
 675         GEN6_RPNSWREQ,                  /* 0xA008 */
 676         GEN6_RC_VIDEO_FREQ,             /* 0xA00C */
 677         RING_TAIL(GEN6_BSD_RING_BASE),  /* 0x12000 (base) */
 678         RING_TAIL(VEBOX_RING_BASE),     /* 0x1a000 (base) */
 679         RING_TAIL(BLT_RING_BASE),       /* 0x22000 (base) */
 680         /* TODO: Other registers are not yet used */
 681 };
 682
 683 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
 684 {
 685         u32 offset = i915_mmio_reg_offset(*reg);
 686
 687         if (key < offset)
 688                 return -1;
 689         else if (key > offset)
 690                 return 1;
 691         else
 692                 return 0;
 693 }
 694
 695 static bool is_gen8_shadowed(u32 offset)
 696 {
 697         const i915_reg_t *regs = gen8_shadowed_regs;
 698
 699         return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
 700                        mmio_reg_cmp);
 701 }
 702
 703 #define __gen8_reg_write_fw_domains(offset) \
 704 ({ \
 705         enum forcewake_domains __fwd; \
 706         if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
 707                 __fwd = FORCEWAKE_RENDER; \
 708         else \
 709                 __fwd = 0; \
 710         __fwd; \
 711 })
 712
 713 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 714 static const struct intel_forcewake_range __chv_fw_ranges[] = {
 715         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 716         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 717         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
 718         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 719         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
 720         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 721         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
 722         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 723         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
 724         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
 725         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
 726         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 727         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 728         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
 729         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
 730         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
 731 };
 732
 733 #define __fwtable_reg_write_fw_domains(offset) \
 734 ({ \
 735         enum forcewake_domains __fwd = 0; \
 736         if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
 737                 __fwd = find_fw_domain(dev_priv, offset); \
 738         __fwd; \
 739 })
 740
 741 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 742 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
 743         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
 744         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
 745         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
 746         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
 747         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
 748         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
 749         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
 750         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
 751         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
 752         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
 753         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
 754         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
 755         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
 756         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
 757         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
 758         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
 759         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
 760         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 761         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
 762         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
 763         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
 764         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
 765         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
 766         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
 767         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
 768         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 769         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
 770         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
 771         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
 772         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
 773         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
 774         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 775 };
 776
 777 static void
 778 ilk_dummy_write(struct drm_i915_private *dev_priv)
 779 {
 780         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 781          * the chip from rc6 before touching it for real. MI_MODE is masked,
 782          * hence harmless to write 0 into. */
 783         __raw_i915_write32(dev_priv, MI_MODE, 0);
 784 }
 785
 786 static void
 787 __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 788                       const i915_reg_t reg,
 789                       const bool read,
 790                       const bool before)
 791 {
 792         if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
 793                  "Unclaimed %s register 0x%x\n",
 794                  read ? "read from" : "write to",
 795                  i915_mmio_reg_offset(reg)))
 796                 i915.mmio_debug--; /* Only report the first N failures */
 797 }
 798
 799 static inline void
 800 unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 801                     const i915_reg_t reg,
 802                     const bool read,
 803                     const bool before)
 804 {
 805         if (likely(!i915.mmio_debug))
 806                 return;
 807
 808         __unclaimed_reg_debug(dev_priv, reg, read, before);
 809 }
 810
 811 static const enum decoupled_power_domain fw2dpd_domain[] = {
 812         GEN9_DECOUPLED_PD_RENDER,
 813         GEN9_DECOUPLED_PD_BLITTER,
 814         GEN9_DECOUPLED_PD_ALL,
 815         GEN9_DECOUPLED_PD_MEDIA,
 816         GEN9_DECOUPLED_PD_ALL,
 817         GEN9_DECOUPLED_PD_ALL,
 818         GEN9_DECOUPLED_PD_ALL
 819 };
 820
 821 /*
 822  * Decoupled MMIO access for only 1 DWORD
 823  */
 824 static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
 825                                          u32 reg,
 826                                          enum forcewake_domains fw_domain,
 827                                          enum decoupled_ops operation)
 828 {
 829         enum decoupled_power_domain dp_domain;
 830         u32 ctrl_reg_data = 0;
 831
 832         dp_domain = fw2dpd_domain[fw_domain - 1];
 833
 834         ctrl_reg_data |= reg;
 835         ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
 836         ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
 837         ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
 838         __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);
 839
 840         if (wait_for_atomic((__raw_i915_read32(dev_priv,
 841                             GEN9_DECOUPLED_REG0_DW1) &
 842                             GEN9_DECOUPLED_DW1_GO) == 0,
 843                             FORCEWAKE_ACK_TIMEOUT_MS))
 844                 DRM_ERROR("Decoupled MMIO wait timed out\n");
 845 }
 846
 847 static inline u32
 848 __gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
 849                              u32 reg,
 850                              enum forcewake_domains fw_domain)
 851 {
 852         __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
 853                                      GEN9_DECOUPLED_OP_READ);
 854
 855         return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
 856 }
 857
 858 static inline void
 859 __gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
 860                             u32 reg, u32 data,
 861                             enum forcewake_domains fw_domain)
 862 {
 863
 864         __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);
 865
 866         __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
 867                                      GEN9_DECOUPLED_OP_WRITE);
 868 }
 869
 870
 871 #define GEN2_READ_HEADER(x) \
 872         u##x val = 0; \
 873         assert_rpm_wakelock_held(dev_priv);
 874
 875 #define GEN2_READ_FOOTER \
 876         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 877         return val
 878
 879 #define __gen2_read(x) \
 880 static u##x \
 881 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 882         GEN2_READ_HEADER(x); \
 883         val = __raw_i915_read##x(dev_priv, reg); \
 884         GEN2_READ_FOOTER; \
 885 }
 886
 887 #define __gen5_read(x) \
 888 static u##x \
 889 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 890         GEN2_READ_HEADER(x); \
 891         ilk_dummy_write(dev_priv); \
 892         val = __raw_i915_read##x(dev_priv, reg); \
 893         GEN2_READ_FOOTER; \
 894 }
 895
 896 __gen5_read(8)
 897 __gen5_read(16)
 898 __gen5_read(32)
 899 __gen5_read(64)
 900 __gen2_read(8)
 901 __gen2_read(16)
 902 __gen2_read(32)
 903 __gen2_read(64)
 904
 905 #undef __gen5_read
 906 #undef __gen2_read
 907
 908 #undef GEN2_READ_FOOTER
 909 #undef GEN2_READ_HEADER
 910
 911 #define GEN6_READ_HEADER(x) \
 912         u32 offset = i915_mmio_reg_offset(reg); \
 913         unsigned long irqflags; \
 914         u##x val = 0; \
 915         assert_rpm_wakelock_held(dev_priv); \
 916         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
 917         unclaimed_reg_debug(dev_priv, reg, true, true)
 918
 919 #define GEN6_READ_FOOTER \
 920         unclaimed_reg_debug(dev_priv, reg, true, false); \
 921         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 922         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 923         return val
 924
 925 static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
 926                                         enum forcewake_domains fw_domains)
 927 {
 928         struct intel_uncore_forcewake_domain *domain;
 929
 930         for_each_fw_domain_masked(domain, fw_domains, dev_priv)
 931                 fw_domain_arm_timer(domain);
 932
 933         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
 934         dev_priv->uncore.fw_domains_active |= fw_domains;
 935 }
 936
 937 static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
 938                                      enum forcewake_domains fw_domains)
 939 {
 940         if (WARN_ON(!fw_domains))
 941                 return;
 942
 943         /* Turn on all requested but inactive supported forcewake domains. */
 944         fw_domains &= dev_priv->uncore.fw_domains;
 945         fw_domains &= ~dev_priv->uncore.fw_domains_active;
 946
 947         if (fw_domains)
 948                 ___force_wake_auto(dev_priv, fw_domains);
 949 }
 950
 951 #define __gen_read(func, x) \
 952 static u##x \
 953 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 954         enum forcewake_domains fw_engine; \
 955         GEN6_READ_HEADER(x); \
 956         fw_engine = __##func##_reg_read_fw_domains(offset); \
 957         if (fw_engine) \
 958                 __force_wake_auto(dev_priv, fw_engine); \
 959         val = __raw_i915_read##x(dev_priv, reg); \
 960         GEN6_READ_FOOTER; \
 961 }
 962 #define __gen6_read(x) __gen_read(gen6, x)
 963 #define __fwtable_read(x) __gen_read(fwtable, x)
 964
 965 #define __gen9_decoupled_read(x) \
 966 static u##x \
 967 gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
 968                        i915_reg_t reg, bool trace) { \
 969         enum forcewake_domains fw_engine; \
 970         GEN6_READ_HEADER(x); \
 971         fw_engine = __fwtable_reg_read_fw_domains(offset); \
 972         if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
 973                 unsigned i; \
 974                 u32 *ptr_data = (u32 *) &val; \
 975                 for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
 976                         *ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
 977                                                                  offset, \
 978                                                                  fw_engine); \
 979         } else { \
 980                 val = __raw_i915_read##x(dev_priv, reg); \
 981         } \
 982         GEN6_READ_FOOTER; \
 983 }
 984
 985 __gen9_decoupled_read(32)
 986 __gen9_decoupled_read(64)
 987 __fwtable_read(8)
 988 __fwtable_read(16)
 989 __fwtable_read(32)
 990 __fwtable_read(64)
 991 __gen6_read(8)
 992 __gen6_read(16)
 993 __gen6_read(32)
 994 __gen6_read(64)
 995
 996 #undef __fwtable_read
 997 #undef __gen6_read
 998 #undef GEN6_READ_FOOTER
 999 #undef GEN6_READ_HEADER
1000
1001 #define GEN2_WRITE_HEADER \
1002         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1003         assert_rpm_wakelock_held(dev_priv); \
1004
1005 #define GEN2_WRITE_FOOTER
1006
1007 #define __gen2_write(x) \
1008 static void \
1009 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1010         GEN2_WRITE_HEADER; \
1011         __raw_i915_write##x(dev_priv, reg, val); \
1012         GEN2_WRITE_FOOTER; \
1013 }
1014
1015 #define __gen5_write(x) \
1016 static void \
1017 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1018         GEN2_WRITE_HEADER; \
1019         ilk_dummy_write(dev_priv); \
1020         __raw_i915_write##x(dev_priv, reg, val); \
1021         GEN2_WRITE_FOOTER; \
1022 }
1023
1024 __gen5_write(8)
1025 __gen5_write(16)
1026 __gen5_write(32)
1027 __gen2_write(8)
1028 __gen2_write(16)
1029 __gen2_write(32)
1030
1031 #undef __gen5_write
1032 #undef __gen2_write
1033
1034 #undef GEN2_WRITE_FOOTER
1035 #undef GEN2_WRITE_HEADER
1036
1037 #define GEN6_WRITE_HEADER \
1038         u32 offset = i915_mmio_reg_offset(reg); \
1039         unsigned long irqflags; \
1040         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1041         assert_rpm_wakelock_held(dev_priv); \
1042         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1043         unclaimed_reg_debug(dev_priv, reg, false, true)
1044
1045 #define GEN6_WRITE_FOOTER \
1046         unclaimed_reg_debug(dev_priv, reg, false, false); \
1047         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
1048
1049 #define __gen6_write(x) \
1050 static void \
1051 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1052         u32 __fifo_ret = 0; \
1053         GEN6_WRITE_HEADER; \
1054         if (NEEDS_FORCE_WAKE(offset)) { \
1055                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
1056         } \
1057         __raw_i915_write##x(dev_priv, reg, val); \
1058         if (unlikely(__fifo_ret)) { \
1059                 gen6_gt_check_fifodbg(dev_priv); \
1060         } \
1061         GEN6_WRITE_FOOTER; \
1062 }
1063
1064 #define __gen_write(func, x) \
1065 static void \
1066 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1067         enum forcewake_domains fw_engine; \
1068         GEN6_WRITE_HEADER; \
1069         fw_engine = __##func##_reg_write_fw_domains(offset); \
1070         if (fw_engine) \
1071                 __force_wake_auto(dev_priv, fw_engine); \
1072         __raw_i915_write##x(dev_priv, reg, val); \
1073         GEN6_WRITE_FOOTER; \
1074 }
1075 #define __gen8_write(x) __gen_write(gen8, x)
1076 #define __fwtable_write(x) __gen_write(fwtable, x)
1077
1078 #define __gen9_decoupled_write(x) \
1079 static void \
1080 gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
1081                         i915_reg_t reg, u##x val, \
1082                 bool trace) { \
1083         enum forcewake_domains fw_engine; \
1084         GEN6_WRITE_HEADER; \
1085         fw_engine = __fwtable_reg_write_fw_domains(offset); \
1086         if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
1087                 __gen9_decoupled_mmio_write(dev_priv, \
1088                                             offset, \
1089                                             val, \
1090                                             fw_engine); \
1091         else \
1092                 __raw_i915_write##x(dev_priv, reg, val); \
1093         GEN6_WRITE_FOOTER; \
1094 }
1095
1096 __gen9_decoupled_write(32)
1097 __fwtable_write(8)
1098 __fwtable_write(16)
1099 __fwtable_write(32)
1100 __gen8_write(8)
1101 __gen8_write(16)
1102 __gen8_write(32)
1103 __gen6_write(8)
1104 __gen6_write(16)
1105 __gen6_write(32)
1106
1107 #undef __fwtable_write
1108 #undef __gen8_write
1109 #undef __gen6_write
1110 #undef GEN6_WRITE_FOOTER
1111 #undef GEN6_WRITE_HEADER
1112
1113 #define ASSIGN_WRITE_MMIO_VFUNCS(x) \
1114 do { \
1115         dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
1116         dev_priv->uncore.funcs.mmio_writew = x##_write16; \
1117         dev_priv->uncore.funcs.mmio_writel = x##_write32; \
1118 } while (0)
1119
1120 #define ASSIGN_READ_MMIO_VFUNCS(x) \
1121 do { \
1122         dev_priv->uncore.funcs.mmio_readb = x##_read8; \
1123         dev_priv->uncore.funcs.mmio_readw = x##_read16; \
1124         dev_priv->uncore.funcs.mmio_readl = x##_read32; \
1125         dev_priv->uncore.funcs.mmio_readq = x##_read64; \
1126 } while (0)
1127
1128
1129 static void fw_domain_init(struct drm_i915_private *dev_priv,
1130                            enum forcewake_domain_id domain_id,
1131                            i915_reg_t reg_set,
1132                            i915_reg_t reg_ack)
1133 {
1134         struct intel_uncore_forcewake_domain *d;
1135
1136         if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
1137                 return;
1138
1139         d = &dev_priv->uncore.fw_domain[domain_id];
1140
1141         WARN_ON(d->wake_count);
1142
1143         d->wake_count = 0;
1144         d->reg_set = reg_set;
1145         d->reg_ack = reg_ack;
1146
1147         if (IS_GEN6(dev_priv)) {
1148                 d->val_reset = 0;
1149                 d->val_set = FORCEWAKE_KERNEL;
1150                 d->val_clear = 0;
1151         } else {
1152                 /* WaRsClearFWBitsAtReset:bdw,skl */
1153                 d->val_reset = _MASKED_BIT_DISABLE(0xffff);
1154                 d->val_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
1155                 d->val_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
1156         }
1157
1158         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1159                 d->reg_post = FORCEWAKE_ACK_VLV;
1160         else if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv) || IS_GEN8(dev_priv))
1161                 d->reg_post = ECOBUS;
1162
1163         d->i915 = dev_priv;
1164         d->id = domain_id;
1165
1166         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
1167         BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
1168         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
1169
1170         d->mask = 1 << domain_id;
1171
1172         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1173         d->timer.function = intel_uncore_fw_release_timer;
1174
1175         dev_priv->uncore.fw_domains |= (1 << domain_id);
1176
1177         fw_domain_reset(d);
1178 }
1179
1180 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1181 {
1182         if (INTEL_INFO(dev_priv)->gen <= 5)
1183                 return;
1184
1185         if (IS_GEN9(dev_priv)) {
1186                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1187                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1188                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1189                                FORCEWAKE_RENDER_GEN9,
1190                                FORCEWAKE_ACK_RENDER_GEN9);
1191                 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
1192                                FORCEWAKE_BLITTER_GEN9,
1193                                FORCEWAKE_ACK_BLITTER_GEN9);
1194                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1195                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
1196         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1197                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1198                 if (!IS_CHERRYVIEW(dev_priv))
1199                         dev_priv->uncore.funcs.force_wake_put =
1200                                 fw_domains_put_with_fifo;
1201                 else
1202                         dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1203                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1204                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
1205                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1206                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
1207         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1208                 dev_priv->uncore.funcs.force_wake_get =
1209                         fw_domains_get_with_thread_status;
1210                 if (IS_HASWELL(dev_priv))
1211                         dev_priv->uncore.funcs.force_wake_put =
1212                                 fw_domains_put_with_fifo;
1213                 else
1214                         dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1215                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1216                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
1217         } else if (IS_IVYBRIDGE(dev_priv)) {
1218                 u32 ecobus;
1219
1220                 /* IVB configs may use multi-threaded forcewake */
1221
1222                 /* A small trick here - if the bios hasn't configured
1223                  * MT forcewake, and if the device is in RC6, then
1224                  * force_wake_mt_get will not wake the device and the
1225                  * ECOBUS read will return zero. Which will be
1226                  * (correctly) interpreted by the test below as MT
1227                  * forcewake being disabled.
1228                  */
1229                 dev_priv->uncore.funcs.force_wake_get =
1230                         fw_domains_get_with_thread_status;
1231                 dev_priv->uncore.funcs.force_wake_put =
1232                         fw_domains_put_with_fifo;
1233
1234                 /* We need to init first for ECOBUS access and then
1235                  * determine later if we want to reinit, in case of MT access is
1236                  * not working. In this stage we don't know which flavour this
1237                  * ivb is, so it is better to reset also the gen6 fw registers
1238                  * before the ecobus check.
1239                  */
1240
1241                 __raw_i915_write32(dev_priv, FORCEWAKE, 0);
1242                 __raw_posting_read(dev_priv, ECOBUS);
1243
1244                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1245                                FORCEWAKE_MT, FORCEWAKE_MT_ACK);
1246
1247                 spin_lock_irq(&dev_priv->uncore.lock);
1248                 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_ALL);
1249                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1250                 fw_domains_put_with_fifo(dev_priv, FORCEWAKE_ALL);
1251                 spin_unlock_irq(&dev_priv->uncore.lock);
1252
1253                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
1254                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
1255                         DRM_INFO("when using vblank-synced partial screen updates.\n");
1256                         fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1257                                        FORCEWAKE, FORCEWAKE_ACK);
1258                 }
1259         } else if (IS_GEN6(dev_priv)) {
1260                 dev_priv->uncore.funcs.force_wake_get =
1261                         fw_domains_get_with_thread_status;
1262                 dev_priv->uncore.funcs.force_wake_put =
1263                         fw_domains_put_with_fifo;
1264                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1265                                FORCEWAKE, FORCEWAKE_ACK);
1266         }
1267
1268         if (intel_vgpu_active(dev_priv)) {
1269                 dev_priv->uncore.funcs.force_wake_get = vgpu_fw_domains_nop;
1270                 dev_priv->uncore.funcs.force_wake_put = vgpu_fw_domains_nop;
1271         }
1272
1273         /* All future platforms are expected to require complex power gating */
1274         WARN_ON(dev_priv->uncore.fw_domains == 0);
1275 }
1276
1277 #define ASSIGN_FW_DOMAINS_TABLE(d) \
1278 { \
1279         dev_priv->uncore.fw_domains_table = \
1280                         (struct intel_forcewake_range *)(d); \
1281         dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
1282 }
1283
1284 void intel_uncore_init(struct drm_i915_private *dev_priv)
1285 {
1286         i915_check_vgpu(dev_priv);
1287
1288         intel_uncore_edram_detect(dev_priv);
1289         intel_uncore_fw_domains_init(dev_priv);
1290         __intel_uncore_early_sanitize(dev_priv, false);
1291
1292         dev_priv->uncore.unclaimed_mmio_check = 1;
1293
1294         switch (INTEL_INFO(dev_priv)->gen) {
1295         default:
1296         case 9:
1297                 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1298                 ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
1299                 ASSIGN_READ_MMIO_VFUNCS(fwtable);
1300                 if (HAS_DECOUPLED_MMIO(dev_priv)) {
1301                         dev_priv->uncore.funcs.mmio_readl =
1302                                                 gen9_decoupled_read32;
1303                         dev_priv->uncore.funcs.mmio_readq =
1304                                                 gen9_decoupled_read64;
1305                         dev_priv->uncore.funcs.mmio_writel =
1306                                                 gen9_decoupled_write32;
1307                 }
1308                 break;
1309         case 8:
1310                 if (IS_CHERRYVIEW(dev_priv)) {
1311                         ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1312                         ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
1313                         ASSIGN_READ_MMIO_VFUNCS(fwtable);
1314
1315                 } else {
1316                         ASSIGN_WRITE_MMIO_VFUNCS(gen8);
1317                         ASSIGN_READ_MMIO_VFUNCS(gen6);
1318                 }
1319                 break;
1320         case 7:
1321         case 6:
1322                 ASSIGN_WRITE_MMIO_VFUNCS(gen6);
1323
1324                 if (IS_VALLEYVIEW(dev_priv)) {
1325                         ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1326                         ASSIGN_READ_MMIO_VFUNCS(fwtable);
1327                 } else {
1328                         ASSIGN_READ_MMIO_VFUNCS(gen6);
1329                 }
1330                 break;
1331         case 5:
1332                 ASSIGN_WRITE_MMIO_VFUNCS(gen5);
1333                 ASSIGN_READ_MMIO_VFUNCS(gen5);
1334                 break;
1335         case 4:
1336         case 3:
1337         case 2:
1338                 ASSIGN_WRITE_MMIO_VFUNCS(gen2);
1339                 ASSIGN_READ_MMIO_VFUNCS(gen2);
1340                 break;
1341         }
1342
1343         i915_check_and_clear_faults(dev_priv);
1344 }
1345 #undef ASSIGN_WRITE_MMIO_VFUNCS
1346 #undef ASSIGN_READ_MMIO_VFUNCS
1347
1348 void intel_uncore_fini(struct drm_i915_private *dev_priv)
1349 {
1350         /* Paranoia: make sure we have disabled everything before we exit. */
1351         intel_uncore_sanitize(dev_priv);
1352         intel_uncore_forcewake_reset(dev_priv, false);
1353 }
1354
1355 #define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1)
1356
1357 static const struct register_whitelist {
1358         i915_reg_t offset_ldw, offset_udw;
1359         uint32_t size;
1360         /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
1361         uint32_t gen_bitmask;
1362 } whitelist[] = {
1363         { .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
1364           .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
1365           .size = 8, .gen_bitmask = GEN_RANGE(4, 9) },
1366 };
1367
1368 int i915_reg_read_ioctl(struct drm_device *dev,
1369                         void *data, struct drm_file *file)
1370 {
1371         struct drm_i915_private *dev_priv = to_i915(dev);
1372         struct drm_i915_reg_read *reg = data;
1373         struct register_whitelist const *entry = whitelist;
1374         unsigned size;
1375         i915_reg_t offset_ldw, offset_udw;
1376         int i, ret = 0;
1377
1378         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
1379                 if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
1380                     (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask))
1381                         break;
1382         }
1383
1384         if (i == ARRAY_SIZE(whitelist))
1385                 return -EINVAL;
1386
1387         /* We use the low bits to encode extra flags as the register should
1388          * be naturally aligned (and those that are not so aligned merely
1389          * limit the available flags for that register).
1390          */
1391         offset_ldw = entry->offset_ldw;
1392         offset_udw = entry->offset_udw;
1393         size = entry->size;
1394         size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw);
1395
1396         intel_runtime_pm_get(dev_priv);
1397
1398         switch (size) {
1399         case 8 | 1:
1400                 reg->val = I915_READ64_2x32(offset_ldw, offset_udw);
1401                 break;
1402         case 8:
1403                 reg->val = I915_READ64(offset_ldw);
1404                 break;
1405         case 4:
1406                 reg->val = I915_READ(offset_ldw);
1407                 break;
1408         case 2:
1409                 reg->val = I915_READ16(offset_ldw);
1410                 break;
1411         case 1:
1412                 reg->val = I915_READ8(offset_ldw);
1413                 break;
1414         default:
1415                 ret = -EINVAL;
1416                 goto out;
1417         }
1418
1419 out:
1420         intel_runtime_pm_put(dev_priv);
1421         return ret;
1422 }
1423
1424 static int i915_reset_complete(struct pci_dev *pdev)
1425 {
1426         u8 gdrst;
1427         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
1428         return (gdrst & GRDOM_RESET_STATUS) == 0;
1429 }
1430
1431 static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1432 {
1433         struct pci_dev *pdev = dev_priv->drm.pdev;
1434
1435         /* assert reset for at least 20 usec */
1436         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1437         udelay(20);
1438         pci_write_config_byte(pdev, I915_GDRST, 0);
1439
1440         return wait_for(i915_reset_complete(pdev), 500);
1441 }
1442
1443 static int g4x_reset_complete(struct pci_dev *pdev)
1444 {
1445         u8 gdrst;
1446         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
1447         return (gdrst & GRDOM_RESET_ENABLE) == 0;
1448 }
1449
1450 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1451 {
1452         struct pci_dev *pdev = dev_priv->drm.pdev;
1453         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1454         return wait_for(g4x_reset_complete(pdev), 500);
1455 }
1456
1457 static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1458 {
1459         struct pci_dev *pdev = dev_priv->drm.pdev;
1460         int ret;
1461
1462         pci_write_config_byte(pdev, I915_GDRST,
1463                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1464         ret =  wait_for(g4x_reset_complete(pdev), 500);
1465         if (ret)
1466                 return ret;
1467
1468         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1469         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1470         POSTING_READ(VDECCLK_GATE_D);
1471
1472         pci_write_config_byte(pdev, I915_GDRST,
1473                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1474         ret =  wait_for(g4x_reset_complete(pdev), 500);
1475         if (ret)
1476                 return ret;
1477
1478         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1479         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1480         POSTING_READ(VDECCLK_GATE_D);
1481
1482         pci_write_config_byte(pdev, I915_GDRST, 0);
1483
1484         return 0;
1485 }
1486
1487 static int ironlake_do_reset(struct drm_i915_private *dev_priv,
1488                              unsigned engine_mask)
1489 {
1490         int ret;
1491
1492         I915_WRITE(ILK_GDSR,
1493                    ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1494         ret = intel_wait_for_register(dev_priv,
1495                                       ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
1496                                       500);
1497         if (ret)
1498                 return ret;
1499
1500         I915_WRITE(ILK_GDSR,
1501                    ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1502         ret = intel_wait_for_register(dev_priv,
1503                                       ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
1504                                       500);
1505         if (ret)
1506                 return ret;
1507
1508         I915_WRITE(ILK_GDSR, 0);
1509
1510         return 0;
1511 }
1512
1513 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
1514 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
1515                                 u32 hw_domain_mask)
1516 {
1517         /* GEN6_GDRST is not in the gt power well, no need to check
1518          * for fifo space for the write or forcewake the chip for
1519          * the read
1520          */
1521         __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
1522
1523         /* Spin waiting for the device to ack the reset requests */
1524         return intel_wait_for_register_fw(dev_priv,
1525                                           GEN6_GDRST, hw_domain_mask, 0,
1526                                           500);
1527 }
1528
1529 /**
1530  * gen6_reset_engines - reset individual engines
1531  * @dev_priv: i915 device
1532  * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
1533  *
1534  * This function will reset the individual engines that are set in engine_mask.
1535  * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
1536  *
1537  * Note: It is responsibility of the caller to handle the difference between
1538  * asking full domain reset versus reset for all available individual engines.
1539  *
1540  * Returns 0 on success, nonzero on error.
1541  */
1542 static int gen6_reset_engines(struct drm_i915_private *dev_priv,
1543                               unsigned engine_mask)
1544 {
1545         struct intel_engine_cs *engine;
1546         const u32 hw_engine_mask[I915_NUM_ENGINES] = {
1547                 [RCS] = GEN6_GRDOM_RENDER,
1548                 [BCS] = GEN6_GRDOM_BLT,
1549                 [VCS] = GEN6_GRDOM_MEDIA,
1550                 [VCS2] = GEN8_GRDOM_MEDIA2,
1551                 [VECS] = GEN6_GRDOM_VECS,
1552         };
1553         u32 hw_mask;
1554         int ret;
1555
1556         if (engine_mask == ALL_ENGINES) {
1557                 hw_mask = GEN6_GRDOM_FULL;
1558         } else {
1559                 unsigned int tmp;
1560
1561                 hw_mask = 0;
1562                 for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1563                         hw_mask |= hw_engine_mask[engine->id];
1564         }
1565
1566         ret = gen6_hw_domain_reset(dev_priv, hw_mask);
1567
1568         intel_uncore_forcewake_reset(dev_priv, true);
1569
1570         return ret;
1571 }
1572
1573 /**
1574  * intel_wait_for_register_fw - wait until register matches expected state
1575  * @dev_priv: the i915 device
1576  * @reg: the register to read
1577  * @mask: mask to apply to register value
1578  * @value: expected value
1579  * @timeout_ms: timeout in millisecond
1580  *
1581  * This routine waits until the target register @reg contains the expected
1582  * @value after applying the @mask, i.e. it waits until ::
1583  *
1584  *     (I915_READ_FW(reg) & mask) == value
1585  *
1586  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
1587  *
1588  * Note that this routine assumes the caller holds forcewake asserted, it is
1589  * not suitable for very long waits. See intel_wait_for_register() if you
1590  * wish to wait without holding forcewake for the duration (i.e. you expect
1591  * the wait to be slow).
1592  *
1593  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1594  */
1595 int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
1596                                i915_reg_t reg,
1597                                const u32 mask,
1598                                const u32 value,
1599                                const unsigned long timeout_ms)
1600 {
1601 #define done ((I915_READ_FW(reg) & mask) == value)
1602         int ret = wait_for_us(done, 2);
1603         if (ret)
1604                 ret = wait_for(done, timeout_ms);
1605         return ret;
1606 #undef done
1607 }
1608
1609 /**
1610  * intel_wait_for_register - wait until register matches expected state
1611  * @dev_priv: the i915 device
1612  * @reg: the register to read
1613  * @mask: mask to apply to register value
1614  * @value: expected value
1615  * @timeout_ms: timeout in millisecond
1616  *
1617  * This routine waits until the target register @reg contains the expected
1618  * @value after applying the @mask, i.e. it waits until ::
1619  *
1620  *     (I915_READ(reg) & mask) == value
1621  *
1622  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
1623  *
1624  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1625  */
1626 int intel_wait_for_register(struct drm_i915_private *dev_priv,
1627                             i915_reg_t reg,
1628                             const u32 mask,
1629                             const u32 value,
1630                             const unsigned long timeout_ms)
1631 {
1632
1633         unsigned fw =
1634                 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
1635         int ret;
1636
1637         intel_uncore_forcewake_get(dev_priv, fw);
1638         ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2);
1639         intel_uncore_forcewake_put(dev_priv, fw);
1640         if (ret)
1641                 ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
1642                                timeout_ms);
1643
1644         return ret;
1645 }
1646
1647 static int gen8_request_engine_reset(struct intel_engine_cs *engine)
1648 {
1649         struct drm_i915_private *dev_priv = engine->i915;
1650         int ret;
1651
1652         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1653                       _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
1654
1655         ret = intel_wait_for_register_fw(dev_priv,
1656                                          RING_RESET_CTL(engine->mmio_base),
1657                                          RESET_CTL_READY_TO_RESET,
1658                                          RESET_CTL_READY_TO_RESET,
1659                                          700);
1660         if (ret)
1661                 DRM_ERROR("%s: reset request timeout\n", engine->name);
1662
1663         return ret;
1664 }
1665
1666 static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine)
1667 {
1668         struct drm_i915_private *dev_priv = engine->i915;
1669
1670         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1671                       _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
1672 }
1673
1674 static int gen8_reset_engines(struct drm_i915_private *dev_priv,
1675                               unsigned engine_mask)
1676 {
1677         struct intel_engine_cs *engine;
1678         unsigned int tmp;
1679
1680         for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1681                 if (gen8_request_engine_reset(engine))
1682                         goto not_ready;
1683
1684         return gen6_reset_engines(dev_priv, engine_mask);
1685
1686 not_ready:
1687         for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1688                 gen8_unrequest_engine_reset(engine);
1689
1690         return -EIO;
1691 }
1692
1693 typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);
1694
1695 static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1696 {
1697         if (!i915.reset)
1698                 return NULL;
1699
1700         if (INTEL_INFO(dev_priv)->gen >= 8)
1701                 return gen8_reset_engines;
1702         else if (INTEL_INFO(dev_priv)->gen >= 6)
1703                 return gen6_reset_engines;
1704         else if (IS_GEN5(dev_priv))
1705                 return ironlake_do_reset;
1706         else if (IS_G4X(dev_priv))
1707                 return g4x_do_reset;
1708         else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
1709                 return g33_do_reset;
1710         else if (INTEL_INFO(dev_priv)->gen >= 3)
1711                 return i915_do_reset;
1712         else
1713                 return NULL;
1714 }
1715
1716 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1717 {
1718         reset_func reset;
1719         int ret;
1720
1721         reset = intel_get_gpu_reset(dev_priv);
1722         if (reset == NULL)
1723                 return -ENODEV;
1724
1725         /* If the power well sleeps during the reset, the reset
1726          * request may be dropped and never completes (causing -EIO).
1727          */
1728         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1729         ret = reset(dev_priv, engine_mask);
1730         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1731
1732         return ret;
1733 }
1734
1735 bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1736 {
1737         return intel_get_gpu_reset(dev_priv) != NULL;
1738 }
1739
1740 int intel_guc_reset(struct drm_i915_private *dev_priv)
1741 {
1742         int ret;
1743         unsigned long irqflags;
1744
1745         if (!HAS_GUC(dev_priv))
1746                 return -EINVAL;
1747
1748         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1749         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
1750
1751         ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
1752
1753         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
1754         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1755
1756         return ret;
1757 }
1758
1759 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
1760 {
1761         return check_for_unclaimed_mmio(dev_priv);
1762 }
1763
1764 bool
1765 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
1766 {
1767         if (unlikely(i915.mmio_debug ||
1768                      dev_priv->uncore.unclaimed_mmio_check <= 0))
1769                 return false;
1770
1771         if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
1772                 DRM_DEBUG("Unclaimed register detected, "
1773                           "enabling oneshot unclaimed register reporting. "
1774                           "Please use i915.mmio_debug=N for more information.\n");
1775                 i915.mmio_debug++;
1776                 dev_priv->uncore.unclaimed_mmio_check--;
1777                 return true;
1778         }
1779
1780         return false;
1781 }
1782
1783 static enum forcewake_domains
1784 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
1785                                 i915_reg_t reg)
1786 {
1787         u32 offset = i915_mmio_reg_offset(reg);
1788         enum forcewake_domains fw_domains;
1789
1790         if (HAS_FWTABLE(dev_priv)) {
1791                 fw_domains = __fwtable_reg_read_fw_domains(offset);
1792         } else if (INTEL_GEN(dev_priv) >= 6) {
1793                 fw_domains = __gen6_reg_read_fw_domains(offset);
1794         } else {
1795                 WARN_ON(!IS_GEN(dev_priv, 2, 5));
1796                 fw_domains = 0;
1797         }
1798
1799         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
1800
1801         return fw_domains;
1802 }
1803
1804 static enum forcewake_domains
1805 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
1806                                  i915_reg_t reg)
1807 {
1808         u32 offset = i915_mmio_reg_offset(reg);
1809         enum forcewake_domains fw_domains;
1810
1811         if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
1812                 fw_domains = __fwtable_reg_write_fw_domains(offset);
1813         } else if (IS_GEN8(dev_priv)) {
1814                 fw_domains = __gen8_reg_write_fw_domains(offset);
1815         } else if (IS_GEN(dev_priv, 6, 7)) {
1816                 fw_domains = FORCEWAKE_RENDER;
1817         } else {
1818                 WARN_ON(!IS_GEN(dev_priv, 2, 5));
1819                 fw_domains = 0;
1820         }
1821
1822         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
1823
1824         return fw_domains;
1825 }
1826
1827 /**
1828  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
1829  *                                  a register
1830  * @dev_priv: pointer to struct drm_i915_private
1831  * @reg: register in question
1832  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
1833  *
1834  * Returns a set of forcewake domains required to be taken with for example
1835  * intel_uncore_forcewake_get for the specified register to be accessible in the
1836  * specified mode (read, write or read/write) with raw mmio accessors.
1837  *
1838  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
1839  * callers to do FIFO management on their own or risk losing writes.
1840  */
1841 enum forcewake_domains
1842 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
1843                                i915_reg_t reg, unsigned int op)
1844 {
1845         enum forcewake_domains fw_domains = 0;
1846
1847         WARN_ON(!op);
1848
1849         if (intel_vgpu_active(dev_priv))
1850                 return 0;
1851
1852         if (op & FW_REG_READ)
1853                 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
1854
1855         if (op & FW_REG_WRITE)
1856                 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
1857
1858         return fw_domains;
1859 }
1860
1861 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1862 #include "selftests/intel_uncore.c"
1863 #endif