drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 const char *
  56 intel_display_power_domain_str(enum intel_display_power_domain domain)
  57 {
  58         switch (domain) {
  59         case POWER_DOMAIN_PIPE_A:
  60                 return "PIPE_A";
  61         case POWER_DOMAIN_PIPE_B:
  62                 return "PIPE_B";
  63         case POWER_DOMAIN_PIPE_C:
  64                 return "PIPE_C";
  65         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  66                 return "PIPE_A_PANEL_FITTER";
  67         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  68                 return "PIPE_B_PANEL_FITTER";
  69         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  70                 return "PIPE_C_PANEL_FITTER";
  71         case POWER_DOMAIN_TRANSCODER_A:
  72                 return "TRANSCODER_A";
  73         case POWER_DOMAIN_TRANSCODER_B:
  74                 return "TRANSCODER_B";
  75         case POWER_DOMAIN_TRANSCODER_C:
  76                 return "TRANSCODER_C";
  77         case POWER_DOMAIN_TRANSCODER_EDP:
  78                 return "TRANSCODER_EDP";
  79         case POWER_DOMAIN_TRANSCODER_DSI_A:
  80                 return "TRANSCODER_DSI_A";
  81         case POWER_DOMAIN_TRANSCODER_DSI_C:
  82                 return "TRANSCODER_DSI_C";
  83         case POWER_DOMAIN_PORT_DDI_A_LANES:
  84                 return "PORT_DDI_A_LANES";
  85         case POWER_DOMAIN_PORT_DDI_B_LANES:
  86                 return "PORT_DDI_B_LANES";
  87         case POWER_DOMAIN_PORT_DDI_C_LANES:
  88                 return "PORT_DDI_C_LANES";
  89         case POWER_DOMAIN_PORT_DDI_D_LANES:
  90                 return "PORT_DDI_D_LANES";
  91         case POWER_DOMAIN_PORT_DDI_E_LANES:
  92                 return "PORT_DDI_E_LANES";
  93         case POWER_DOMAIN_PORT_DDI_F_LANES:
  94                 return "PORT_DDI_F_LANES";
  95         case POWER_DOMAIN_PORT_DDI_A_IO:
  96                 return "PORT_DDI_A_IO";
  97         case POWER_DOMAIN_PORT_DDI_B_IO:
  98                 return "PORT_DDI_B_IO";
  99         case POWER_DOMAIN_PORT_DDI_C_IO:
 100                 return "PORT_DDI_C_IO";
 101         case POWER_DOMAIN_PORT_DDI_D_IO:
 102                 return "PORT_DDI_D_IO";
 103         case POWER_DOMAIN_PORT_DDI_E_IO:
 104                 return "PORT_DDI_E_IO";
 105         case POWER_DOMAIN_PORT_DDI_F_IO:
 106                 return "PORT_DDI_F_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_AUX_E:
 128                 return "AUX_E";
 129         case POWER_DOMAIN_AUX_F:
 130                 return "AUX_F";
 131         case POWER_DOMAIN_AUX_IO_A:
 132                 return "AUX_IO_A";
 133         case POWER_DOMAIN_AUX_TBT1:
 134                 return "AUX_TBT1";
 135         case POWER_DOMAIN_AUX_TBT2:
 136                 return "AUX_TBT2";
 137         case POWER_DOMAIN_AUX_TBT3:
 138                 return "AUX_TBT3";
 139         case POWER_DOMAIN_AUX_TBT4:
 140                 return "AUX_TBT4";
 141         case POWER_DOMAIN_GMBUS:
 142                 return "GMBUS";
 143         case POWER_DOMAIN_INIT:
 144                 return "INIT";
 145         case POWER_DOMAIN_MODESET:
 146                 return "MODESET";
 147         case POWER_DOMAIN_GT_IRQ:
 148                 return "GT_IRQ";
 149         default:
 150                 MISSING_CASE(domain);
 151                 return "?";
 152         }
 153 }
 154
 155 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 156                                     struct i915_power_well *power_well)
 157 {
 158         DRM_DEBUG_KMS("enabling %s\n", power_well->desc->name);
 159         power_well->desc->ops->enable(dev_priv, power_well);
 160         power_well->hw_enabled = true;
 161 }
 162
 163 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 164                                      struct i915_power_well *power_well)
 165 {
 166         DRM_DEBUG_KMS("disabling %s\n", power_well->desc->name);
 167         power_well->hw_enabled = false;
 168         power_well->desc->ops->disable(dev_priv, power_well);
 169 }
 170
 171 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 172                                  struct i915_power_well *power_well)
 173 {
 174         if (!power_well->count++)
 175                 intel_power_well_enable(dev_priv, power_well);
 176 }
 177
 178 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 179                                  struct i915_power_well *power_well)
 180 {
 181         WARN(!power_well->count, "Use count on power well %s is already zero",
 182              power_well->desc->name);
 183
 184         if (!--power_well->count)
 185                 intel_power_well_disable(dev_priv, power_well);
 186 }
 187
 188 /**
 189  * __intel_display_power_is_enabled - unlocked check for a power domain
 190  * @dev_priv: i915 device instance
 191  * @domain: power domain to check
 192  *
 193  * This is the unlocked version of intel_display_power_is_enabled() and should
 194  * only be used from error capture and recovery code where deadlocks are
 195  * possible.
 196  *
 197  * Returns:
 198  * True when the power domain is enabled, false otherwise.
 199  */
 200 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 201                                       enum intel_display_power_domain domain)
 202 {
 203         struct i915_power_well *power_well;
 204         bool is_enabled;
 205
 206         if (dev_priv->runtime_pm.suspended)
 207                 return false;
 208
 209         is_enabled = true;
 210
 211         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 212                 if (power_well->desc->always_on)
 213                         continue;
 214
 215                 if (!power_well->hw_enabled) {
 216                         is_enabled = false;
 217                         break;
 218                 }
 219         }
 220
 221         return is_enabled;
 222 }
 223
 224 /**
 225  * intel_display_power_is_enabled - check for a power domain
 226  * @dev_priv: i915 device instance
 227  * @domain: power domain to check
 228  *
 229  * This function can be used to check the hw power domain state. It is mostly
 230  * used in hardware state readout functions. Everywhere else code should rely
 231  * upon explicit power domain reference counting to ensure that the hardware
 232  * block is powered up before accessing it.
 233  *
 234  * Callers must hold the relevant modesetting locks to ensure that concurrent
 235  * threads can't disable the power well while the caller tries to read a few
 236  * registers.
 237  *
 238  * Returns:
 239  * True when the power domain is enabled, false otherwise.
 240  */
 241 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 242                                     enum intel_display_power_domain domain)
 243 {
 244         struct i915_power_domains *power_domains;
 245         bool ret;
 246
 247         power_domains = &dev_priv->power_domains;
 248
 249         mutex_lock(&power_domains->lock);
 250         ret = __intel_display_power_is_enabled(dev_priv, domain);
 251         mutex_unlock(&power_domains->lock);
 252
 253         return ret;
 254 }
 255
 256 /*
 257  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 258  * when not needed anymore. We have 4 registers that can request the power well
 259  * to be enabled, and it will only be disabled if none of the registers is
 260  * requesting it to be enabled.
 261  */
 262 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 263                                        u8 irq_pipe_mask, bool has_vga)
 264 {
 265         struct pci_dev *pdev = dev_priv->drm.pdev;
 266
 267         /*
 268          * After we re-enable the power well, if we touch VGA register 0x3d5
 269          * we'll get unclaimed register interrupts. This stops after we write
 270          * anything to the VGA MSR register. The vgacon module uses this
 271          * register all the time, so if we unbind our driver and, as a
 272          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 273          * console_unlock(). So make here we touch the VGA MSR register, making
 274          * sure vgacon can keep working normally without triggering interrupts
 275          * and error messages.
 276          */
 277         if (has_vga) {
 278                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 279                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 280                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 281         }
 282
 283         if (irq_pipe_mask)
 284                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 285 }
 286
 287 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 288                                        u8 irq_pipe_mask)
 289 {
 290         if (irq_pipe_mask)
 291                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 292 }
 293
 294
 295 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 296                                            struct i915_power_well *power_well)
 297 {
 298         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 299         int pw_idx = power_well->desc->hsw.idx;
 300
 301         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 302         WARN_ON(intel_wait_for_register(dev_priv,
 303                                         regs->driver,
 304                                         HSW_PWR_WELL_CTL_STATE(pw_idx),
 305                                         HSW_PWR_WELL_CTL_STATE(pw_idx),
 306                                         1));
 307 }
 308
 309 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 310                                      const struct i915_power_well_regs *regs,
 311                                      int pw_idx)
 312 {
 313         u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 314         u32 ret;
 315
 316         ret = I915_READ(regs->bios) & req_mask ? 1 : 0;
 317         ret |= I915_READ(regs->driver) & req_mask ? 2 : 0;
 318         if (regs->kvmr.reg)
 319                 ret |= I915_READ(regs->kvmr) & req_mask ? 4 : 0;
 320         ret |= I915_READ(regs->debug) & req_mask ? 8 : 0;
 321
 322         return ret;
 323 }
 324
 325 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 326                                             struct i915_power_well *power_well)
 327 {
 328         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 329         int pw_idx = power_well->desc->hsw.idx;
 330         bool disabled;
 331         u32 reqs;
 332
 333         /*
 334          * Bspec doesn't require waiting for PWs to get disabled, but still do
 335          * this for paranoia. The known cases where a PW will be forced on:
 336          * - a KVMR request on any power well via the KVMR request register
 337          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 338          *   DEBUG request registers
 339          * Skip the wait in case any of the request bits are set and print a
 340          * diagnostic message.
 341          */
 342         wait_for((disabled = !(I915_READ(regs->driver) &
 343                                HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 344                  (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
 345         if (disabled)
 346                 return;
 347
 348         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 349                       power_well->desc->name,
 350                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 351 }
 352
 353 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 354                                            enum skl_power_gate pg)
 355 {
 356         /* Timeout 5us for PG#0, for other PGs 1us */
 357         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 358                                         SKL_FUSE_PG_DIST_STATUS(pg),
 359                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 360 }
 361
 362 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 363                                   struct i915_power_well *power_well)
 364 {
 365         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 366         int pw_idx = power_well->desc->hsw.idx;
 367         bool wait_fuses = power_well->desc->hsw.has_fuses;
 368         enum skl_power_gate uninitialized_var(pg);
 369         u32 val;
 370
 371         if (wait_fuses) {
 372                 pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 373                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 374                 /*
 375                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 376                  * before enabling the power well and PW1/PG1's own fuse
 377                  * state after the enabling. For all other power wells with
 378                  * fuses we only have to wait for that PW/PG's fuse state
 379                  * after the enabling.
 380                  */
 381                 if (pg == SKL_PG1)
 382                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 383         }
 384
 385         val = I915_READ(regs->driver);
 386         I915_WRITE(regs->driver, val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 387         hsw_wait_for_power_well_enable(dev_priv, power_well);
 388
 389         /* Display WA #1178: cnl */
 390         if (IS_CANNONLAKE(dev_priv) &&
 391             pw_idx >= GLK_PW_CTL_IDX_AUX_B &&
 392             pw_idx <= CNL_PW_CTL_IDX_AUX_F) {
 393                 val = I915_READ(CNL_AUX_ANAOVRD1(pw_idx));
 394                 val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS;
 395                 I915_WRITE(CNL_AUX_ANAOVRD1(pw_idx), val);
 396         }
 397
 398         if (wait_fuses)
 399                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 400
 401         hsw_power_well_post_enable(dev_priv,
 402                                    power_well->desc->hsw.irq_pipe_mask,
 403                                    power_well->desc->hsw.has_vga);
 404 }
 405
 406 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 407                                    struct i915_power_well *power_well)
 408 {
 409         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 410         int pw_idx = power_well->desc->hsw.idx;
 411         u32 val;
 412
 413         hsw_power_well_pre_disable(dev_priv,
 414                                    power_well->desc->hsw.irq_pipe_mask);
 415
 416         val = I915_READ(regs->driver);
 417         I915_WRITE(regs->driver, val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 418         hsw_wait_for_power_well_disable(dev_priv, power_well);
 419 }
 420
 421 #define ICL_AUX_PW_TO_PORT(pw_idx)      ((pw_idx) - ICL_PW_CTL_IDX_AUX_A)
 422
 423 static void
 424 icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 425                                     struct i915_power_well *power_well)
 426 {
 427         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 428         int pw_idx = power_well->desc->hsw.idx;
 429         enum port port = ICL_AUX_PW_TO_PORT(pw_idx);
 430         u32 val;
 431
 432         val = I915_READ(regs->driver);
 433         I915_WRITE(regs->driver, val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 434
 435         val = I915_READ(ICL_PORT_CL_DW12(port));
 436         I915_WRITE(ICL_PORT_CL_DW12(port), val | ICL_LANE_ENABLE_AUX);
 437
 438         hsw_wait_for_power_well_enable(dev_priv, power_well);
 439
 440         /* Display WA #1178: icl */
 441         if (IS_ICELAKE(dev_priv) &&
 442             pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 443             !intel_bios_is_port_edp(dev_priv, port)) {
 444                 val = I915_READ(ICL_AUX_ANAOVRD1(pw_idx));
 445                 val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS;
 446                 I915_WRITE(ICL_AUX_ANAOVRD1(pw_idx), val);
 447         }
 448 }
 449
 450 static void
 451 icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 452                                      struct i915_power_well *power_well)
 453 {
 454         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 455         int pw_idx = power_well->desc->hsw.idx;
 456         enum port port = ICL_AUX_PW_TO_PORT(pw_idx);
 457         u32 val;
 458
 459         val = I915_READ(ICL_PORT_CL_DW12(port));
 460         I915_WRITE(ICL_PORT_CL_DW12(port), val & ~ICL_LANE_ENABLE_AUX);
 461
 462         val = I915_READ(regs->driver);
 463         I915_WRITE(regs->driver, val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 464
 465         hsw_wait_for_power_well_disable(dev_priv, power_well);
 466 }
 467
 468 /*
 469  * We should only use the power well if we explicitly asked the hardware to
 470  * enable it, so check if it's enabled and also check if we've requested it to
 471  * be enabled.
 472  */
 473 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 474                                    struct i915_power_well *power_well)
 475 {
 476         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 477         int pw_idx = power_well->desc->hsw.idx;
 478         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 479                    HSW_PWR_WELL_CTL_STATE(pw_idx);
 480
 481         return (I915_READ(regs->driver) & mask) == mask;
 482 }
 483
 484 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 485 {
 486         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 487                   "DC9 already programmed to be enabled.\n");
 488         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 489                   "DC5 still not disabled to enable DC9.\n");
 490         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL2) &
 491                   HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 492                   "Power well 2 on.\n");
 493         WARN_ONCE(intel_irqs_enabled(dev_priv),
 494                   "Interrupts not disabled yet.\n");
 495
 496          /*
 497           * TODO: check for the following to verify the conditions to enter DC9
 498           * state are satisfied:
 499           * 1] Check relevant display engine registers to verify if mode set
 500           * disable sequence was followed.
 501           * 2] Check if display uninitialize sequence is initialized.
 502           */
 503 }
 504
 505 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 506 {
 507         WARN_ONCE(intel_irqs_enabled(dev_priv),
 508                   "Interrupts not disabled yet.\n");
 509         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 510                   "DC5 still not disabled.\n");
 511
 512          /*
 513           * TODO: check for the following to verify DC9 state was indeed
 514           * entered before programming to disable it:
 515           * 1] Check relevant display engine registers to verify if mode
 516           *  set disable sequence was followed.
 517           * 2] Check if display uninitialize sequence is initialized.
 518           */
 519 }
 520
 521 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 522                                 u32 state)
 523 {
 524         int rewrites = 0;
 525         int rereads = 0;
 526         u32 v;
 527
 528         I915_WRITE(DC_STATE_EN, state);
 529
 530         /* It has been observed that disabling the dc6 state sometimes
 531          * doesn't stick and dmc keeps returning old value. Make sure
 532          * the write really sticks enough times and also force rewrite until
 533          * we are confident that state is exactly what we want.
 534          */
 535         do  {
 536                 v = I915_READ(DC_STATE_EN);
 537
 538                 if (v != state) {
 539                         I915_WRITE(DC_STATE_EN, state);
 540                         rewrites++;
 541                         rereads = 0;
 542                 } else if (rereads++ > 5) {
 543                         break;
 544                 }
 545
 546         } while (rewrites < 100);
 547
 548         if (v != state)
 549                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 550                           state, v);
 551
 552         /* Most of the times we need one retry, avoid spam */
 553         if (rewrites > 1)
 554                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 555                               state, rewrites);
 556 }
 557
 558 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 559 {
 560         u32 mask;
 561
 562         mask = DC_STATE_EN_UPTO_DC5;
 563         if (INTEL_GEN(dev_priv) >= 11)
 564                 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 565         else if (IS_GEN9_LP(dev_priv))
 566                 mask |= DC_STATE_EN_DC9;
 567         else
 568                 mask |= DC_STATE_EN_UPTO_DC6;
 569
 570         return mask;
 571 }
 572
 573 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 574 {
 575         u32 val;
 576
 577         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 578
 579         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 580                       dev_priv->csr.dc_state, val);
 581         dev_priv->csr.dc_state = val;
 582 }
 583
 584 /**
 585  * gen9_set_dc_state - set target display C power state
 586  * @dev_priv: i915 device instance
 587  * @state: target DC power state
 588  * - DC_STATE_DISABLE
 589  * - DC_STATE_EN_UPTO_DC5
 590  * - DC_STATE_EN_UPTO_DC6
 591  * - DC_STATE_EN_DC9
 592  *
 593  * Signal to DMC firmware/HW the target DC power state passed in @state.
 594  * DMC/HW can turn off individual display clocks and power rails when entering
 595  * a deeper DC power state (higher in number) and turns these back when exiting
 596  * that state to a shallower power state (lower in number). The HW will decide
 597  * when to actually enter a given state on an on-demand basis, for instance
 598  * depending on the active state of display pipes. The state of display
 599  * registers backed by affected power rails are saved/restored as needed.
 600  *
 601  * Based on the above enabling a deeper DC power state is asynchronous wrt.
 602  * enabling it. Disabling a deeper power state is synchronous: for instance
 603  * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 604  * back on and register state is restored. This is guaranteed by the MMIO write
 605  * to DC_STATE_EN blocking until the state is restored.
 606  */
 607 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 608 {
 609         uint32_t val;
 610         uint32_t mask;
 611
 612         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 613                 state &= dev_priv->csr.allowed_dc_mask;
 614
 615         val = I915_READ(DC_STATE_EN);
 616         mask = gen9_dc_mask(dev_priv);
 617         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 618                       val & mask, state);
 619
 620         /* Check if DMC is ignoring our DC state requests */
 621         if ((val & mask) != dev_priv->csr.dc_state)
 622                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 623                           dev_priv->csr.dc_state, val & mask);
 624
 625         val &= ~mask;
 626         val |= state;
 627
 628         gen9_write_dc_state(dev_priv, val);
 629
 630         dev_priv->csr.dc_state = val & mask;
 631 }
 632
 633 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 634 {
 635         assert_can_enable_dc9(dev_priv);
 636
 637         DRM_DEBUG_KMS("Enabling DC9\n");
 638         /*
 639          * Power sequencer reset is not needed on
 640          * platforms with South Display Engine on PCH,
 641          * because PPS registers are always on.
 642          */
 643         if (!HAS_PCH_SPLIT(dev_priv))
 644                 intel_power_sequencer_reset(dev_priv);
 645         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 646 }
 647
 648 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 649 {
 650         assert_can_disable_dc9(dev_priv);
 651
 652         DRM_DEBUG_KMS("Disabling DC9\n");
 653
 654         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 655
 656         intel_pps_unlock_regs_wa(dev_priv);
 657 }
 658
 659 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 660 {
 661         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 662                   "CSR program storage start is NULL\n");
 663         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 664         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 665 }
 666
 667 static struct i915_power_well *
 668 lookup_power_well(struct drm_i915_private *dev_priv,
 669                   enum i915_power_well_id power_well_id)
 670 {
 671         struct i915_power_well *power_well;
 672
 673         for_each_power_well(dev_priv, power_well)
 674                 if (power_well->desc->id == power_well_id)
 675                         return power_well;
 676
 677         /*
 678          * It's not feasible to add error checking code to the callers since
 679          * this condition really shouldn't happen and it doesn't even make sense
 680          * to abort things like display initialization sequences. Just return
 681          * the first power well and hope the WARN gets reported so we can fix
 682          * our driver.
 683          */
 684         WARN(1, "Power well %d not defined for this platform\n", power_well_id);
 685         return &dev_priv->power_domains.power_wells[0];
 686 }
 687
 688 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 689 {
 690         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 691                                         SKL_DISP_PW_2);
 692
 693         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 694
 695         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 696                   "DC5 already programmed to be enabled.\n");
 697         assert_rpm_wakelock_held(dev_priv);
 698
 699         assert_csr_loaded(dev_priv);
 700 }
 701
 702 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 703 {
 704         assert_can_enable_dc5(dev_priv);
 705
 706         DRM_DEBUG_KMS("Enabling DC5\n");
 707
 708         /* Wa Display #1183: skl,kbl,cfl */
 709         if (IS_GEN9_BC(dev_priv))
 710                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 711                            SKL_SELECT_ALTERNATE_DC_EXIT);
 712
 713         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 714 }
 715
 716 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 717 {
 718         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 719                   "Backlight is not disabled.\n");
 720         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 721                   "DC6 already programmed to be enabled.\n");
 722
 723         assert_csr_loaded(dev_priv);
 724 }
 725
 726 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 727 {
 728         assert_can_enable_dc6(dev_priv);
 729
 730         DRM_DEBUG_KMS("Enabling DC6\n");
 731
 732         /* Wa Display #1183: skl,kbl,cfl */
 733         if (IS_GEN9_BC(dev_priv))
 734                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 735                            SKL_SELECT_ALTERNATE_DC_EXIT);
 736
 737         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 738 }
 739
 740 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 741                                    struct i915_power_well *power_well)
 742 {
 743         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 744         int pw_idx = power_well->desc->hsw.idx;
 745         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 746         u32 bios_req = I915_READ(regs->bios);
 747
 748         /* Take over the request bit if set by BIOS. */
 749         if (bios_req & mask) {
 750                 u32 drv_req = I915_READ(regs->driver);
 751
 752                 if (!(drv_req & mask))
 753                         I915_WRITE(regs->driver, drv_req | mask);
 754                 I915_WRITE(regs->bios, bios_req & ~mask);
 755         }
 756 }
 757
 758 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 759                                            struct i915_power_well *power_well)
 760 {
 761         bxt_ddi_phy_init(dev_priv, power_well->desc->bxt.phy);
 762 }
 763
 764 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 765                                             struct i915_power_well *power_well)
 766 {
 767         bxt_ddi_phy_uninit(dev_priv, power_well->desc->bxt.phy);
 768 }
 769
 770 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 771                                             struct i915_power_well *power_well)
 772 {
 773         return bxt_ddi_phy_is_enabled(dev_priv, power_well->desc->bxt.phy);
 774 }
 775
 776 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 777 {
 778         struct i915_power_well *power_well;
 779
 780         power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
 781         if (power_well->count > 0)
 782                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
 783
 784         power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
 785         if (power_well->count > 0)
 786                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
 787
 788         if (IS_GEMINILAKE(dev_priv)) {
 789                 power_well = lookup_power_well(dev_priv,
 790                                                GLK_DISP_PW_DPIO_CMN_C);
 791                 if (power_well->count > 0)
 792                         bxt_ddi_phy_verify_state(dev_priv,
 793                                                  power_well->desc->bxt.phy);
 794         }
 795 }
 796
 797 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 798                                            struct i915_power_well *power_well)
 799 {
 800         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 801 }
 802
 803 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 804 {
 805         u32 tmp = I915_READ(DBUF_CTL);
 806
 807         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 808              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 809              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 810 }
 811
 812 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 813                                           struct i915_power_well *power_well)
 814 {
 815         struct intel_cdclk_state cdclk_state = {};
 816
 817         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 818
 819         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 820         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
 821         WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_state));
 822
 823         gen9_assert_dbuf_enabled(dev_priv);
 824
 825         if (IS_GEN9_LP(dev_priv))
 826                 bxt_verify_ddi_phy_power_wells(dev_priv);
 827 }
 828
 829 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 830                                            struct i915_power_well *power_well)
 831 {
 832         if (!dev_priv->csr.dmc_payload)
 833                 return;
 834
 835         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 836                 skl_enable_dc6(dev_priv);
 837         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 838                 gen9_enable_dc5(dev_priv);
 839 }
 840
 841 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 842                                          struct i915_power_well *power_well)
 843 {
 844 }
 845
 846 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 847                                            struct i915_power_well *power_well)
 848 {
 849 }
 850
 851 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 852                                              struct i915_power_well *power_well)
 853 {
 854         return true;
 855 }
 856
 857 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 858                                          struct i915_power_well *power_well)
 859 {
 860         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 861                 i830_enable_pipe(dev_priv, PIPE_A);
 862         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 863                 i830_enable_pipe(dev_priv, PIPE_B);
 864 }
 865
 866 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 867                                           struct i915_power_well *power_well)
 868 {
 869         i830_disable_pipe(dev_priv, PIPE_B);
 870         i830_disable_pipe(dev_priv, PIPE_A);
 871 }
 872
 873 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 874                                           struct i915_power_well *power_well)
 875 {
 876         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 877                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 878 }
 879
 880 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 881                                           struct i915_power_well *power_well)
 882 {
 883         if (power_well->count > 0)
 884                 i830_pipes_power_well_enable(dev_priv, power_well);
 885         else
 886                 i830_pipes_power_well_disable(dev_priv, power_well);
 887 }
 888
 889 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 890                                struct i915_power_well *power_well, bool enable)
 891 {
 892         int pw_idx = power_well->desc->vlv.idx;
 893         u32 mask;
 894         u32 state;
 895         u32 ctrl;
 896
 897         mask = PUNIT_PWRGT_MASK(pw_idx);
 898         state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
 899                          PUNIT_PWRGT_PWR_GATE(pw_idx);
 900
 901         mutex_lock(&dev_priv->pcu_lock);
 902
 903 #define COND \
 904         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 905
 906         if (COND)
 907                 goto out;
 908
 909         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 910         ctrl &= ~mask;
 911         ctrl |= state;
 912         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 913
 914         if (wait_for(COND, 100))
 915                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 916                           state,
 917                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 918
 919 #undef COND
 920
 921 out:
 922         mutex_unlock(&dev_priv->pcu_lock);
 923 }
 924
 925 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 926                                   struct i915_power_well *power_well)
 927 {
 928         vlv_set_power_well(dev_priv, power_well, true);
 929 }
 930
 931 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 932                                    struct i915_power_well *power_well)
 933 {
 934         vlv_set_power_well(dev_priv, power_well, false);
 935 }
 936
 937 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 938                                    struct i915_power_well *power_well)
 939 {
 940         int pw_idx = power_well->desc->vlv.idx;
 941         bool enabled = false;
 942         u32 mask;
 943         u32 state;
 944         u32 ctrl;
 945
 946         mask = PUNIT_PWRGT_MASK(pw_idx);
 947         ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
 948
 949         mutex_lock(&dev_priv->pcu_lock);
 950
 951         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 952         /*
 953          * We only ever set the power-on and power-gate states, anything
 954          * else is unexpected.
 955          */
 956         WARN_ON(state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
 957                 state != PUNIT_PWRGT_PWR_GATE(pw_idx));
 958         if (state == ctrl)
 959                 enabled = true;
 960
 961         /*
 962          * A transient state at this point would mean some unexpected party
 963          * is poking at the power controls too.
 964          */
 965         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 966         WARN_ON(ctrl != state);
 967
 968         mutex_unlock(&dev_priv->pcu_lock);
 969
 970         return enabled;
 971 }
 972
 973 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 974 {
 975         u32 val;
 976
 977         /*
 978          * On driver load, a pipe may be active and driving a DSI display.
 979          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 980          * (and never recovering) in this case. intel_dsi_post_disable() will
 981          * clear it when we turn off the display.
 982          */
 983         val = I915_READ(DSPCLK_GATE_D);
 984         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 985         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 986         I915_WRITE(DSPCLK_GATE_D, val);
 987
 988         /*
 989          * Disable trickle feed and enable pnd deadline calculation
 990          */
 991         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 992         I915_WRITE(CBR1_VLV, 0);
 993
 994         WARN_ON(dev_priv->rawclk_freq == 0);
 995
 996         I915_WRITE(RAWCLK_FREQ_VLV,
 997                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 998 }
 999
1000 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1001 {
1002         struct intel_encoder *encoder;
1003         enum pipe pipe;
1004
1005         /*
1006          * Enable the CRI clock source so we can get at the
1007          * display and the reference clock for VGA
1008          * hotplug / manual detection. Supposedly DSI also
1009          * needs the ref clock up and running.
1010          *
1011          * CHV DPLL B/C have some issues if VGA mode is enabled.
1012          */
1013         for_each_pipe(dev_priv, pipe) {
1014                 u32 val = I915_READ(DPLL(pipe));
1015
1016                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1017                 if (pipe != PIPE_A)
1018                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1019
1020                 I915_WRITE(DPLL(pipe), val);
1021         }
1022
1023         vlv_init_display_clock_gating(dev_priv);
1024
1025         spin_lock_irq(&dev_priv->irq_lock);
1026         valleyview_enable_display_irqs(dev_priv);
1027         spin_unlock_irq(&dev_priv->irq_lock);
1028
1029         /*
1030          * During driver initialization/resume we can avoid restoring the
1031          * part of the HW/SW state that will be inited anyway explicitly.
1032          */
1033         if (dev_priv->power_domains.initializing)
1034                 return;
1035
1036         intel_hpd_init(dev_priv);
1037
1038         /* Re-enable the ADPA, if we have one */
1039         for_each_intel_encoder(&dev_priv->drm, encoder) {
1040                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1041                         intel_crt_reset(&encoder->base);
1042         }
1043
1044         i915_redisable_vga_power_on(dev_priv);
1045
1046         intel_pps_unlock_regs_wa(dev_priv);
1047 }
1048
1049 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1050 {
1051         spin_lock_irq(&dev_priv->irq_lock);
1052         valleyview_disable_display_irqs(dev_priv);
1053         spin_unlock_irq(&dev_priv->irq_lock);
1054
1055         /* make sure we're done processing display irqs */
1056         synchronize_irq(dev_priv->drm.irq);
1057
1058         intel_power_sequencer_reset(dev_priv);
1059
1060         /* Prevent us from re-enabling polling on accident in late suspend */
1061         if (!dev_priv->drm.dev->power.is_suspended)
1062                 intel_hpd_poll_init(dev_priv);
1063 }
1064
1065 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1066                                           struct i915_power_well *power_well)
1067 {
1068         vlv_set_power_well(dev_priv, power_well, true);
1069
1070         vlv_display_power_well_init(dev_priv);
1071 }
1072
1073 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1074                                            struct i915_power_well *power_well)
1075 {
1076         vlv_display_power_well_deinit(dev_priv);
1077
1078         vlv_set_power_well(dev_priv, power_well, false);
1079 }
1080
1081 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1082                                            struct i915_power_well *power_well)
1083 {
1084         /* since ref/cri clock was enabled */
1085         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1086
1087         vlv_set_power_well(dev_priv, power_well, true);
1088
1089         /*
1090          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1091          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1092          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1093          *   b. The other bits such as sfr settings / modesel may all
1094          *      be set to 0.
1095          *
1096          * This should only be done on init and resume from S3 with
1097          * both PLLs disabled, or we risk losing DPIO and PLL
1098          * synchronization.
1099          */
1100         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1101 }
1102
1103 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1104                                             struct i915_power_well *power_well)
1105 {
1106         enum pipe pipe;
1107
1108         for_each_pipe(dev_priv, pipe)
1109                 assert_pll_disabled(dev_priv, pipe);
1110
1111         /* Assert common reset */
1112         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1113
1114         vlv_set_power_well(dev_priv, power_well, false);
1115 }
1116
1117 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1118
1119 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1120
1121 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1122 {
1123         struct i915_power_well *cmn_bc =
1124                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1125         struct i915_power_well *cmn_d =
1126                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1127         u32 phy_control = dev_priv->chv_phy_control;
1128         u32 phy_status = 0;
1129         u32 phy_status_mask = 0xffffffff;
1130
1131         /*
1132          * The BIOS can leave the PHY is some weird state
1133          * where it doesn't fully power down some parts.
1134          * Disable the asserts until the PHY has been fully
1135          * reset (ie. the power well has been disabled at
1136          * least once).
1137          */
1138         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1139                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1140                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1141                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1142                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1143                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1144                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1145
1146         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1147                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1148                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1149                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1150
1151         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
1152                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1153
1154                 /* this assumes override is only used to enable lanes */
1155                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1156                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1157
1158                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1159                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1160
1161                 /* CL1 is on whenever anything is on in either channel */
1162                 if (BITS_SET(phy_control,
1163                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1164                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1165                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1166
1167                 /*
1168                  * The DPLLB check accounts for the pipe B + port A usage
1169                  * with CL2 powered up but all the lanes in the second channel
1170                  * powered down.
1171                  */
1172                 if (BITS_SET(phy_control,
1173                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1174                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1175                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1176
1177                 if (BITS_SET(phy_control,
1178                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1179                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1180                 if (BITS_SET(phy_control,
1181                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1182                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1183
1184                 if (BITS_SET(phy_control,
1185                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1186                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1187                 if (BITS_SET(phy_control,
1188                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1189                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1190         }
1191
1192         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
1193                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1194
1195                 /* this assumes override is only used to enable lanes */
1196                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1197                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1198
1199                 if (BITS_SET(phy_control,
1200                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1201                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1202
1203                 if (BITS_SET(phy_control,
1204                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1205                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1206                 if (BITS_SET(phy_control,
1207                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1208                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1209         }
1210
1211         phy_status &= phy_status_mask;
1212
1213         /*
1214          * The PHY may be busy with some initial calibration and whatnot,
1215          * so the power state can take a while to actually change.
1216          */
1217         if (intel_wait_for_register(dev_priv,
1218                                     DISPLAY_PHY_STATUS,
1219                                     phy_status_mask,
1220                                     phy_status,
1221                                     10))
1222                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1223                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1224                            phy_status, dev_priv->chv_phy_control);
1225 }
1226
1227 #undef BITS_SET
1228
1229 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1230                                            struct i915_power_well *power_well)
1231 {
1232         enum dpio_phy phy;
1233         enum pipe pipe;
1234         uint32_t tmp;
1235
1236         WARN_ON_ONCE(power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1237                      power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1238
1239         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1240                 pipe = PIPE_A;
1241                 phy = DPIO_PHY0;
1242         } else {
1243                 pipe = PIPE_C;
1244                 phy = DPIO_PHY1;
1245         }
1246
1247         /* since ref/cri clock was enabled */
1248         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1249         vlv_set_power_well(dev_priv, power_well, true);
1250
1251         /* Poll for phypwrgood signal */
1252         if (intel_wait_for_register(dev_priv,
1253                                     DISPLAY_PHY_STATUS,
1254                                     PHY_POWERGOOD(phy),
1255                                     PHY_POWERGOOD(phy),
1256                                     1))
1257                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1258
1259         mutex_lock(&dev_priv->sb_lock);
1260
1261         /* Enable dynamic power down */
1262         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1263         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1264                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1265         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1266
1267         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1268                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1269                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1270                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1271         } else {
1272                 /*
1273                  * Force the non-existing CL2 off. BXT does this
1274                  * too, so maybe it saves some power even though
1275                  * CL2 doesn't exist?
1276                  */
1277                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1278                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1279                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1280         }
1281
1282         mutex_unlock(&dev_priv->sb_lock);
1283
1284         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1285         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1286
1287         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1288                       phy, dev_priv->chv_phy_control);
1289
1290         assert_chv_phy_status(dev_priv);
1291 }
1292
1293 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1294                                             struct i915_power_well *power_well)
1295 {
1296         enum dpio_phy phy;
1297
1298         WARN_ON_ONCE(power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1299                      power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1300
1301         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1302                 phy = DPIO_PHY0;
1303                 assert_pll_disabled(dev_priv, PIPE_A);
1304                 assert_pll_disabled(dev_priv, PIPE_B);
1305         } else {
1306                 phy = DPIO_PHY1;
1307                 assert_pll_disabled(dev_priv, PIPE_C);
1308         }
1309
1310         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1311         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1312
1313         vlv_set_power_well(dev_priv, power_well, false);
1314
1315         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1316                       phy, dev_priv->chv_phy_control);
1317
1318         /* PHY is fully reset now, so we can enable the PHY state asserts */
1319         dev_priv->chv_phy_assert[phy] = true;
1320
1321         assert_chv_phy_status(dev_priv);
1322 }
1323
1324 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1325                                      enum dpio_channel ch, bool override, unsigned int mask)
1326 {
1327         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1328         u32 reg, val, expected, actual;
1329
1330         /*
1331          * The BIOS can leave the PHY is some weird state
1332          * where it doesn't fully power down some parts.
1333          * Disable the asserts until the PHY has been fully
1334          * reset (ie. the power well has been disabled at
1335          * least once).
1336          */
1337         if (!dev_priv->chv_phy_assert[phy])
1338                 return;
1339
1340         if (ch == DPIO_CH0)
1341                 reg = _CHV_CMN_DW0_CH0;
1342         else
1343                 reg = _CHV_CMN_DW6_CH1;
1344
1345         mutex_lock(&dev_priv->sb_lock);
1346         val = vlv_dpio_read(dev_priv, pipe, reg);
1347         mutex_unlock(&dev_priv->sb_lock);
1348
1349         /*
1350          * This assumes !override is only used when the port is disabled.
1351          * All lanes should power down even without the override when
1352          * the port is disabled.
1353          */
1354         if (!override || mask == 0xf) {
1355                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1356                 /*
1357                  * If CH1 common lane is not active anymore
1358                  * (eg. for pipe B DPLL) the entire channel will
1359                  * shut down, which causes the common lane registers
1360                  * to read as 0. That means we can't actually check
1361                  * the lane power down status bits, but as the entire
1362                  * register reads as 0 it's a good indication that the
1363                  * channel is indeed entirely powered down.
1364                  */
1365                 if (ch == DPIO_CH1 && val == 0)
1366                         expected = 0;
1367         } else if (mask != 0x0) {
1368                 expected = DPIO_ANYDL_POWERDOWN;
1369         } else {
1370                 expected = 0;
1371         }
1372
1373         if (ch == DPIO_CH0)
1374                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1375         else
1376                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1377         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1378
1379         WARN(actual != expected,
1380              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1381              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1382              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1383              reg, val);
1384 }
1385
1386 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1387                           enum dpio_channel ch, bool override)
1388 {
1389         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1390         bool was_override;
1391
1392         mutex_lock(&power_domains->lock);
1393
1394         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1395
1396         if (override == was_override)
1397                 goto out;
1398
1399         if (override)
1400                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1401         else
1402                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1403
1404         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1405
1406         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1407                       phy, ch, dev_priv->chv_phy_control);
1408
1409         assert_chv_phy_status(dev_priv);
1410
1411 out:
1412         mutex_unlock(&power_domains->lock);
1413
1414         return was_override;
1415 }
1416
1417 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1418                              bool override, unsigned int mask)
1419 {
1420         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1421         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1422         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1423         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1424
1425         mutex_lock(&power_domains->lock);
1426
1427         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1428         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1429
1430         if (override)
1431                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1432         else
1433                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1434
1435         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1436
1437         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1438                       phy, ch, mask, dev_priv->chv_phy_control);
1439
1440         assert_chv_phy_status(dev_priv);
1441
1442         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1443
1444         mutex_unlock(&power_domains->lock);
1445 }
1446
1447 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1448                                         struct i915_power_well *power_well)
1449 {
1450         enum pipe pipe = PIPE_A;
1451         bool enabled;
1452         u32 state, ctrl;
1453
1454         mutex_lock(&dev_priv->pcu_lock);
1455
1456         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1457         /*
1458          * We only ever set the power-on and power-gate states, anything
1459          * else is unexpected.
1460          */
1461         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1462         enabled = state == DP_SSS_PWR_ON(pipe);
1463
1464         /*
1465          * A transient state at this point would mean some unexpected party
1466          * is poking at the power controls too.
1467          */
1468         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1469         WARN_ON(ctrl << 16 != state);
1470
1471         mutex_unlock(&dev_priv->pcu_lock);
1472
1473         return enabled;
1474 }
1475
1476 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1477                                     struct i915_power_well *power_well,
1478                                     bool enable)
1479 {
1480         enum pipe pipe = PIPE_A;
1481         u32 state;
1482         u32 ctrl;
1483
1484         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1485
1486         mutex_lock(&dev_priv->pcu_lock);
1487
1488 #define COND \
1489         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1490
1491         if (COND)
1492                 goto out;
1493
1494         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1495         ctrl &= ~DP_SSC_MASK(pipe);
1496         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1497         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1498
1499         if (wait_for(COND, 100))
1500                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1501                           state,
1502                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1503
1504 #undef COND
1505
1506 out:
1507         mutex_unlock(&dev_priv->pcu_lock);
1508 }
1509
1510 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1511                                        struct i915_power_well *power_well)
1512 {
1513         chv_set_pipe_power_well(dev_priv, power_well, true);
1514
1515         vlv_display_power_well_init(dev_priv);
1516 }
1517
1518 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1519                                         struct i915_power_well *power_well)
1520 {
1521         vlv_display_power_well_deinit(dev_priv);
1522
1523         chv_set_pipe_power_well(dev_priv, power_well, false);
1524 }
1525
1526 static void
1527 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1528                                  enum intel_display_power_domain domain)
1529 {
1530         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1531         struct i915_power_well *power_well;
1532
1533         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1534                 intel_power_well_get(dev_priv, power_well);
1535
1536         power_domains->domain_use_count[domain]++;
1537 }
1538
1539 /**
1540  * intel_display_power_get - grab a power domain reference
1541  * @dev_priv: i915 device instance
1542  * @domain: power domain to reference
1543  *
1544  * This function grabs a power domain reference for @domain and ensures that the
1545  * power domain and all its parents are powered up. Therefore users should only
1546  * grab a reference to the innermost power domain they need.
1547  *
1548  * Any power domain reference obtained by this function must have a symmetric
1549  * call to intel_display_power_put() to release the reference again.
1550  */
1551 void intel_display_power_get(struct drm_i915_private *dev_priv,
1552                              enum intel_display_power_domain domain)
1553 {
1554         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1555
1556         intel_runtime_pm_get(dev_priv);
1557
1558         mutex_lock(&power_domains->lock);
1559
1560         __intel_display_power_get_domain(dev_priv, domain);
1561
1562         mutex_unlock(&power_domains->lock);
1563 }
1564
1565 /**
1566  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1567  * @dev_priv: i915 device instance
1568  * @domain: power domain to reference
1569  *
1570  * This function grabs a power domain reference for @domain and ensures that the
1571  * power domain and all its parents are powered up. Therefore users should only
1572  * grab a reference to the innermost power domain they need.
1573  *
1574  * Any power domain reference obtained by this function must have a symmetric
1575  * call to intel_display_power_put() to release the reference again.
1576  */
1577 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1578                                         enum intel_display_power_domain domain)
1579 {
1580         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1581         bool is_enabled;
1582
1583         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1584                 return false;
1585
1586         mutex_lock(&power_domains->lock);
1587
1588         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1589                 __intel_display_power_get_domain(dev_priv, domain);
1590                 is_enabled = true;
1591         } else {
1592                 is_enabled = false;
1593         }
1594
1595         mutex_unlock(&power_domains->lock);
1596
1597         if (!is_enabled)
1598                 intel_runtime_pm_put(dev_priv);
1599
1600         return is_enabled;
1601 }
1602
1603 /**
1604  * intel_display_power_put - release a power domain reference
1605  * @dev_priv: i915 device instance
1606  * @domain: power domain to reference
1607  *
1608  * This function drops the power domain reference obtained by
1609  * intel_display_power_get() and might power down the corresponding hardware
1610  * block right away if this is the last reference.
1611  */
1612 void intel_display_power_put(struct drm_i915_private *dev_priv,
1613                              enum intel_display_power_domain domain)
1614 {
1615         struct i915_power_domains *power_domains;
1616         struct i915_power_well *power_well;
1617
1618         power_domains = &dev_priv->power_domains;
1619
1620         mutex_lock(&power_domains->lock);
1621
1622         WARN(!power_domains->domain_use_count[domain],
1623              "Use count on domain %s is already zero\n",
1624              intel_display_power_domain_str(domain));
1625         power_domains->domain_use_count[domain]--;
1626
1627         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1628                 intel_power_well_put(dev_priv, power_well);
1629
1630         mutex_unlock(&power_domains->lock);
1631
1632         intel_runtime_pm_put(dev_priv);
1633 }
1634
1635 #define I830_PIPES_POWER_DOMAINS (              \
1636         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1637         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1638         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1639         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1640         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1641         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1642         BIT_ULL(POWER_DOMAIN_INIT))
1643
1644 #define VLV_DISPLAY_POWER_DOMAINS (             \
1645         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1646         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1647         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1648         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1649         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1650         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1651         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1652         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1653         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1654         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1655         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1656         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1657         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1658         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1659         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1660         BIT_ULL(POWER_DOMAIN_INIT))
1661
1662 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1663         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1664         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1665         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1666         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1667         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1668         BIT_ULL(POWER_DOMAIN_INIT))
1669
1670 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1671         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1672         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1673         BIT_ULL(POWER_DOMAIN_INIT))
1674
1675 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1676         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1677         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1678         BIT_ULL(POWER_DOMAIN_INIT))
1679
1680 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1681         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1682         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1683         BIT_ULL(POWER_DOMAIN_INIT))
1684
1685 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1686         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1687         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1688         BIT_ULL(POWER_DOMAIN_INIT))
1689
1690 #define CHV_DISPLAY_POWER_DOMAINS (             \
1691         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1692         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1693         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1694         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1695         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1696         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1697         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1698         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1699         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1700         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1701         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1702         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1703         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1704         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1705         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1706         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1707         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1708         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1709         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1710         BIT_ULL(POWER_DOMAIN_INIT))
1711
1712 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1713         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1714         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1715         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1716         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1717         BIT_ULL(POWER_DOMAIN_INIT))
1718
1719 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1720         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1721         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1722         BIT_ULL(POWER_DOMAIN_INIT))
1723
1724 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1725         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1726         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1727         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1728         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1729         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1730         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1731         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1732         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1733         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1734         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1735         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1736         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1737         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1738         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1739         BIT_ULL(POWER_DOMAIN_INIT))
1740
1741 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1742         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1743         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1744         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1745         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1746         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1747         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1748         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1749         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1750         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1751         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1752         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1753         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1754         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1755         BIT_ULL(POWER_DOMAIN_INIT))
1756
1757 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1758         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1759         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1760         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1761         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1762         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1763         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1764         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1765         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1766         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1767         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1768         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1769         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1770         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1771         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1772         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1773         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1774         BIT_ULL(POWER_DOMAIN_INIT))
1775 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1776         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1777         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1778         BIT_ULL(POWER_DOMAIN_INIT))
1779 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1780         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1781         BIT_ULL(POWER_DOMAIN_INIT))
1782 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1783         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1784         BIT_ULL(POWER_DOMAIN_INIT))
1785 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1786         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1787         BIT_ULL(POWER_DOMAIN_INIT))
1788 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1789         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1790         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1791         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1792         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1793         BIT_ULL(POWER_DOMAIN_INIT))
1794
1795 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1796         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1797         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1798         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1799         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1800         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1801         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1802         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1803         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1804         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1805         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1806         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1807         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1808         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1809         BIT_ULL(POWER_DOMAIN_INIT))
1810 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1811         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1812         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1813         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1814         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1815         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1816         BIT_ULL(POWER_DOMAIN_INIT))
1817 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1818         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1819         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1820         BIT_ULL(POWER_DOMAIN_INIT))
1821 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1822         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1823         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1824         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1825         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1826         BIT_ULL(POWER_DOMAIN_INIT))
1827
1828 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1829         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1830         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1831         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1832         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1833         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1834         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1835         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1836         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1837         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1838         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1839         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1840         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1841         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1842         BIT_ULL(POWER_DOMAIN_INIT))
1843 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1844         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1845 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1846         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1847 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1848         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1849 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1850         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1851         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1852         BIT_ULL(POWER_DOMAIN_INIT))
1853 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1854         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1855         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1856         BIT_ULL(POWER_DOMAIN_INIT))
1857 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1858         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1859         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1860         BIT_ULL(POWER_DOMAIN_INIT))
1861 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1862         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1863         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
1864         BIT_ULL(POWER_DOMAIN_INIT))
1865 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1866         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1867         BIT_ULL(POWER_DOMAIN_INIT))
1868 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1869         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1870         BIT_ULL(POWER_DOMAIN_INIT))
1871 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1872         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1873         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1874         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1875         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1876         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1877         BIT_ULL(POWER_DOMAIN_INIT))
1878
1879 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1880         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1881         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1882         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1883         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1884         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1885         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1886         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1887         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1888         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1889         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1890         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |                \
1891         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1892         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1893         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1894         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
1895         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1896         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1897         BIT_ULL(POWER_DOMAIN_INIT))
1898 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1899         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1900         BIT_ULL(POWER_DOMAIN_INIT))
1901 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1902         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1903         BIT_ULL(POWER_DOMAIN_INIT))
1904 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1905         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1906         BIT_ULL(POWER_DOMAIN_INIT))
1907 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1908         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1909         BIT_ULL(POWER_DOMAIN_INIT))
1910 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1911         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1912         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
1913         BIT_ULL(POWER_DOMAIN_INIT))
1914 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1915         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1916         BIT_ULL(POWER_DOMAIN_INIT))
1917 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1918         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1919         BIT_ULL(POWER_DOMAIN_INIT))
1920 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1921         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1922         BIT_ULL(POWER_DOMAIN_INIT))
1923 #define CNL_DISPLAY_AUX_F_POWER_DOMAINS (               \
1924         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
1925         BIT_ULL(POWER_DOMAIN_INIT))
1926 #define CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS (            \
1927         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
1928         BIT_ULL(POWER_DOMAIN_INIT))
1929 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1930         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1931         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1932         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1933         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1934         BIT_ULL(POWER_DOMAIN_INIT))
1935
1936 /*
1937  * ICL PW_0/PG_0 domains (HW/DMC control):
1938  * - PCI
1939  * - clocks except port PLL
1940  * - central power except FBC
1941  * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
1942  * ICL PW_1/PG_1 domains (HW/DMC control):
1943  * - DBUF function
1944  * - PIPE_A and its planes, except VGA
1945  * - transcoder EDP + PSR
1946  * - transcoder DSI
1947  * - DDI_A
1948  * - FBC
1949  */
1950 #define ICL_PW_4_POWER_DOMAINS (                        \
1951         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1952         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1953         BIT_ULL(POWER_DOMAIN_INIT))
1954         /* VDSC/joining */
1955 #define ICL_PW_3_POWER_DOMAINS (                        \
1956         ICL_PW_4_POWER_DOMAINS |                        \
1957         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1958         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1959         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1960         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1961         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1962         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1963         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1964         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1965         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1966         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1967         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1968         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
1969         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1970         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
1971         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
1972         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1973         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1974         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1975         BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
1976         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
1977         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |                \
1978         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |                \
1979         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |                \
1980         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |                \
1981         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1982         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1983         BIT_ULL(POWER_DOMAIN_INIT))
1984         /*
1985          * - transcoder WD
1986          * - KVMR (HW control)
1987          */
1988 #define ICL_PW_2_POWER_DOMAINS (                        \
1989         ICL_PW_3_POWER_DOMAINS |                        \
1990         BIT_ULL(POWER_DOMAIN_INIT))
1991         /*
1992          * - eDP/DSI VDSC
1993          * - KVMR (HW control)
1994          */
1995 #define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1996         ICL_PW_2_POWER_DOMAINS |                        \
1997         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1998         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1999         BIT_ULL(POWER_DOMAIN_INIT))
2000
2001 #define ICL_DDI_IO_A_POWER_DOMAINS (                    \
2002         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2003 #define ICL_DDI_IO_B_POWER_DOMAINS (                    \
2004         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2005 #define ICL_DDI_IO_C_POWER_DOMAINS (                    \
2006         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2007 #define ICL_DDI_IO_D_POWER_DOMAINS (                    \
2008         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
2009 #define ICL_DDI_IO_E_POWER_DOMAINS (                    \
2010         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
2011 #define ICL_DDI_IO_F_POWER_DOMAINS (                    \
2012         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
2013
2014 #define ICL_AUX_A_IO_POWER_DOMAINS (                    \
2015         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2016         BIT_ULL(POWER_DOMAIN_AUX_A))
2017 #define ICL_AUX_B_IO_POWER_DOMAINS (                    \
2018         BIT_ULL(POWER_DOMAIN_AUX_B))
2019 #define ICL_AUX_C_IO_POWER_DOMAINS (                    \
2020         BIT_ULL(POWER_DOMAIN_AUX_C))
2021 #define ICL_AUX_D_IO_POWER_DOMAINS (                    \
2022         BIT_ULL(POWER_DOMAIN_AUX_D))
2023 #define ICL_AUX_E_IO_POWER_DOMAINS (                    \
2024         BIT_ULL(POWER_DOMAIN_AUX_E))
2025 #define ICL_AUX_F_IO_POWER_DOMAINS (                    \
2026         BIT_ULL(POWER_DOMAIN_AUX_F))
2027 #define ICL_AUX_TBT1_IO_POWER_DOMAINS (                 \
2028         BIT_ULL(POWER_DOMAIN_AUX_TBT1))
2029 #define ICL_AUX_TBT2_IO_POWER_DOMAINS (                 \
2030         BIT_ULL(POWER_DOMAIN_AUX_TBT2))
2031 #define ICL_AUX_TBT3_IO_POWER_DOMAINS (                 \
2032         BIT_ULL(POWER_DOMAIN_AUX_TBT3))
2033 #define ICL_AUX_TBT4_IO_POWER_DOMAINS (                 \
2034         BIT_ULL(POWER_DOMAIN_AUX_TBT4))
2035
2036 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
2037         .sync_hw = i9xx_power_well_sync_hw_noop,
2038         .enable = i9xx_always_on_power_well_noop,
2039         .disable = i9xx_always_on_power_well_noop,
2040         .is_enabled = i9xx_always_on_power_well_enabled,
2041 };
2042
2043 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
2044         .sync_hw = i9xx_power_well_sync_hw_noop,
2045         .enable = chv_pipe_power_well_enable,
2046         .disable = chv_pipe_power_well_disable,
2047         .is_enabled = chv_pipe_power_well_enabled,
2048 };
2049
2050 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
2051         .sync_hw = i9xx_power_well_sync_hw_noop,
2052         .enable = chv_dpio_cmn_power_well_enable,
2053         .disable = chv_dpio_cmn_power_well_disable,
2054         .is_enabled = vlv_power_well_enabled,
2055 };
2056
2057 static const struct i915_power_well_desc i9xx_always_on_power_well[] = {
2058         {
2059                 .name = "always-on",
2060                 .always_on = 1,
2061                 .domains = POWER_DOMAIN_MASK,
2062                 .ops = &i9xx_always_on_power_well_ops,
2063                 .id = DISP_PW_ID_NONE,
2064         },
2065 };
2066
2067 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
2068         .sync_hw = i830_pipes_power_well_sync_hw,
2069         .enable = i830_pipes_power_well_enable,
2070         .disable = i830_pipes_power_well_disable,
2071         .is_enabled = i830_pipes_power_well_enabled,
2072 };
2073
2074 static const struct i915_power_well_desc i830_power_wells[] = {
2075         {
2076                 .name = "always-on",
2077                 .always_on = 1,
2078                 .domains = POWER_DOMAIN_MASK,
2079                 .ops = &i9xx_always_on_power_well_ops,
2080                 .id = DISP_PW_ID_NONE,
2081         },
2082         {
2083                 .name = "pipes",
2084                 .domains = I830_PIPES_POWER_DOMAINS,
2085                 .ops = &i830_pipes_power_well_ops,
2086                 .id = DISP_PW_ID_NONE,
2087         },
2088 };
2089
2090 static const struct i915_power_well_ops hsw_power_well_ops = {
2091         .sync_hw = hsw_power_well_sync_hw,
2092         .enable = hsw_power_well_enable,
2093         .disable = hsw_power_well_disable,
2094         .is_enabled = hsw_power_well_enabled,
2095 };
2096
2097 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
2098         .sync_hw = i9xx_power_well_sync_hw_noop,
2099         .enable = gen9_dc_off_power_well_enable,
2100         .disable = gen9_dc_off_power_well_disable,
2101         .is_enabled = gen9_dc_off_power_well_enabled,
2102 };
2103
2104 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
2105         .sync_hw = i9xx_power_well_sync_hw_noop,
2106         .enable = bxt_dpio_cmn_power_well_enable,
2107         .disable = bxt_dpio_cmn_power_well_disable,
2108         .is_enabled = bxt_dpio_cmn_power_well_enabled,
2109 };
2110
2111 static const struct i915_power_well_regs hsw_power_well_regs = {
2112         .bios   = HSW_PWR_WELL_CTL1,
2113         .driver = HSW_PWR_WELL_CTL2,
2114         .kvmr   = HSW_PWR_WELL_CTL3,
2115         .debug  = HSW_PWR_WELL_CTL4,
2116 };
2117
2118 static const struct i915_power_well_desc hsw_power_wells[] = {
2119         {
2120                 .name = "always-on",
2121                 .always_on = 1,
2122                 .domains = POWER_DOMAIN_MASK,
2123                 .ops = &i9xx_always_on_power_well_ops,
2124                 .id = DISP_PW_ID_NONE,
2125         },
2126         {
2127                 .name = "display",
2128                 .domains = HSW_DISPLAY_POWER_DOMAINS,
2129                 .ops = &hsw_power_well_ops,
2130                 .id = HSW_DISP_PW_GLOBAL,
2131                 {
2132                         .hsw.regs = &hsw_power_well_regs,
2133                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
2134                         .hsw.has_vga = true,
2135                 },
2136         },
2137 };
2138
2139 static const struct i915_power_well_desc bdw_power_wells[] = {
2140         {
2141                 .name = "always-on",
2142                 .always_on = 1,
2143                 .domains = POWER_DOMAIN_MASK,
2144                 .ops = &i9xx_always_on_power_well_ops,
2145                 .id = DISP_PW_ID_NONE,
2146         },
2147         {
2148                 .name = "display",
2149                 .domains = BDW_DISPLAY_POWER_DOMAINS,
2150                 .ops = &hsw_power_well_ops,
2151                 .id = HSW_DISP_PW_GLOBAL,
2152                 {
2153                         .hsw.regs = &hsw_power_well_regs,
2154                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
2155                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2156                         .hsw.has_vga = true,
2157                 },
2158         },
2159 };
2160
2161 static const struct i915_power_well_ops vlv_display_power_well_ops = {
2162         .sync_hw = i9xx_power_well_sync_hw_noop,
2163         .enable = vlv_display_power_well_enable,
2164         .disable = vlv_display_power_well_disable,
2165         .is_enabled = vlv_power_well_enabled,
2166 };
2167
2168 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
2169         .sync_hw = i9xx_power_well_sync_hw_noop,
2170         .enable = vlv_dpio_cmn_power_well_enable,
2171         .disable = vlv_dpio_cmn_power_well_disable,
2172         .is_enabled = vlv_power_well_enabled,
2173 };
2174
2175 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
2176         .sync_hw = i9xx_power_well_sync_hw_noop,
2177         .enable = vlv_power_well_enable,
2178         .disable = vlv_power_well_disable,
2179         .is_enabled = vlv_power_well_enabled,
2180 };
2181
2182 static const struct i915_power_well_desc vlv_power_wells[] = {
2183         {
2184                 .name = "always-on",
2185                 .always_on = 1,
2186                 .domains = POWER_DOMAIN_MASK,
2187                 .ops = &i9xx_always_on_power_well_ops,
2188                 .id = DISP_PW_ID_NONE,
2189         },
2190         {
2191                 .name = "display",
2192                 .domains = VLV_DISPLAY_POWER_DOMAINS,
2193                 .ops = &vlv_display_power_well_ops,
2194                 .id = VLV_DISP_PW_DISP2D,
2195                 {
2196                         .vlv.idx = PUNIT_PWGT_IDX_DISP2D,
2197                 },
2198         },
2199         {
2200                 .name = "dpio-tx-b-01",
2201                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2202                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2203                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2204                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2205                 .ops = &vlv_dpio_power_well_ops,
2206                 .id = DISP_PW_ID_NONE,
2207                 {
2208                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01,
2209                 },
2210         },
2211         {
2212                 .name = "dpio-tx-b-23",
2213                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2214                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2215                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2216                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2217                 .ops = &vlv_dpio_power_well_ops,
2218                 .id = DISP_PW_ID_NONE,
2219                 {
2220                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23,
2221                 },
2222         },
2223         {
2224                 .name = "dpio-tx-c-01",
2225                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2226                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2227                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2228                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2229                 .ops = &vlv_dpio_power_well_ops,
2230                 .id = DISP_PW_ID_NONE,
2231                 {
2232                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01,
2233                 },
2234         },
2235         {
2236                 .name = "dpio-tx-c-23",
2237                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2238                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2239                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2240                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2241                 .ops = &vlv_dpio_power_well_ops,
2242                 .id = DISP_PW_ID_NONE,
2243                 {
2244                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23,
2245                 },
2246         },
2247         {
2248                 .name = "dpio-common",
2249                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2250                 .ops = &vlv_dpio_cmn_power_well_ops,
2251                 .id = VLV_DISP_PW_DPIO_CMN_BC,
2252                 {
2253                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
2254                 },
2255         },
2256 };
2257
2258 static const struct i915_power_well_desc chv_power_wells[] = {
2259         {
2260                 .name = "always-on",
2261                 .always_on = 1,
2262                 .domains = POWER_DOMAIN_MASK,
2263                 .ops = &i9xx_always_on_power_well_ops,
2264                 .id = DISP_PW_ID_NONE,
2265         },
2266         {
2267                 .name = "display",
2268                 /*
2269                  * Pipe A power well is the new disp2d well. Pipe B and C
2270                  * power wells don't actually exist. Pipe A power well is
2271                  * required for any pipe to work.
2272                  */
2273                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2274                 .ops = &chv_pipe_power_well_ops,
2275                 .id = DISP_PW_ID_NONE,
2276         },
2277         {
2278                 .name = "dpio-common-bc",
2279                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2280                 .ops = &chv_dpio_cmn_power_well_ops,
2281                 .id = VLV_DISP_PW_DPIO_CMN_BC,
2282                 {
2283                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
2284                 },
2285         },
2286         {
2287                 .name = "dpio-common-d",
2288                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2289                 .ops = &chv_dpio_cmn_power_well_ops,
2290                 .id = CHV_DISP_PW_DPIO_CMN_D,
2291                 {
2292                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_D,
2293                 },
2294         },
2295 };
2296
2297 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2298                                          enum i915_power_well_id power_well_id)
2299 {
2300         struct i915_power_well *power_well;
2301         bool ret;
2302
2303         power_well = lookup_power_well(dev_priv, power_well_id);
2304         ret = power_well->desc->ops->is_enabled(dev_priv, power_well);
2305
2306         return ret;
2307 }
2308
2309 static const struct i915_power_well_desc skl_power_wells[] = {
2310         {
2311                 .name = "always-on",
2312                 .always_on = 1,
2313                 .domains = POWER_DOMAIN_MASK,
2314                 .ops = &i9xx_always_on_power_well_ops,
2315                 .id = DISP_PW_ID_NONE,
2316         },
2317         {
2318                 .name = "power well 1",
2319                 /* Handled by the DMC firmware */
2320                 .domains = 0,
2321                 .ops = &hsw_power_well_ops,
2322                 .id = SKL_DISP_PW_1,
2323                 {
2324                         .hsw.regs = &hsw_power_well_regs,
2325                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
2326                         .hsw.has_fuses = true,
2327                 },
2328         },
2329         {
2330                 .name = "MISC IO power well",
2331                 /* Handled by the DMC firmware */
2332                 .domains = 0,
2333                 .ops = &hsw_power_well_ops,
2334                 .id = SKL_DISP_PW_MISC_IO,
2335                 {
2336                         .hsw.regs = &hsw_power_well_regs,
2337                         .hsw.idx = SKL_PW_CTL_IDX_MISC_IO,
2338                 },
2339         },
2340         {
2341                 .name = "DC off",
2342                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2343                 .ops = &gen9_dc_off_power_well_ops,
2344                 .id = DISP_PW_ID_NONE,
2345         },
2346         {
2347                 .name = "power well 2",
2348                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2349                 .ops = &hsw_power_well_ops,
2350                 .id = SKL_DISP_PW_2,
2351                 {
2352                         .hsw.regs = &hsw_power_well_regs,
2353                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
2354                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2355                         .hsw.has_vga = true,
2356                         .hsw.has_fuses = true,
2357                 },
2358         },
2359         {
2360                 .name = "DDI A/E IO power well",
2361                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2362                 .ops = &hsw_power_well_ops,
2363                 .id = DISP_PW_ID_NONE,
2364                 {
2365                         .hsw.regs = &hsw_power_well_regs,
2366                         .hsw.idx = SKL_PW_CTL_IDX_DDI_A_E,
2367                 },
2368         },
2369         {
2370                 .name = "DDI B IO power well",
2371                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2372                 .ops = &hsw_power_well_ops,
2373                 .id = DISP_PW_ID_NONE,
2374                 {
2375                         .hsw.regs = &hsw_power_well_regs,
2376                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
2377                 },
2378         },
2379         {
2380                 .name = "DDI C IO power well",
2381                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2382                 .ops = &hsw_power_well_ops,
2383                 .id = DISP_PW_ID_NONE,
2384                 {
2385                         .hsw.regs = &hsw_power_well_regs,
2386                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
2387                 },
2388         },
2389         {
2390                 .name = "DDI D IO power well",
2391                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2392                 .ops = &hsw_power_well_ops,
2393                 .id = DISP_PW_ID_NONE,
2394                 {
2395                         .hsw.regs = &hsw_power_well_regs,
2396                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
2397                 },
2398         },
2399 };
2400
2401 static const struct i915_power_well_desc bxt_power_wells[] = {
2402         {
2403                 .name = "always-on",
2404                 .always_on = 1,
2405                 .domains = POWER_DOMAIN_MASK,
2406                 .ops = &i9xx_always_on_power_well_ops,
2407                 .id = DISP_PW_ID_NONE,
2408         },
2409         {
2410                 .name = "power well 1",
2411                 .domains = 0,
2412                 .ops = &hsw_power_well_ops,
2413                 .id = SKL_DISP_PW_1,
2414                 {
2415                         .hsw.regs = &hsw_power_well_regs,
2416                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
2417                         .hsw.has_fuses = true,
2418                 },
2419         },
2420         {
2421                 .name = "DC off",
2422                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2423                 .ops = &gen9_dc_off_power_well_ops,
2424                 .id = DISP_PW_ID_NONE,
2425         },
2426         {
2427                 .name = "power well 2",
2428                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2429                 .ops = &hsw_power_well_ops,
2430                 .id = SKL_DISP_PW_2,
2431                 {
2432                         .hsw.regs = &hsw_power_well_regs,
2433                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
2434                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2435                         .hsw.has_vga = true,
2436                         .hsw.has_fuses = true,
2437                 },
2438         },
2439         {
2440                 .name = "dpio-common-a",
2441                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2442                 .ops = &bxt_dpio_cmn_power_well_ops,
2443                 .id = BXT_DISP_PW_DPIO_CMN_A,
2444                 {
2445                         .bxt.phy = DPIO_PHY1,
2446                 },
2447         },
2448         {
2449                 .name = "dpio-common-bc",
2450                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2451                 .ops = &bxt_dpio_cmn_power_well_ops,
2452                 .id = VLV_DISP_PW_DPIO_CMN_BC,
2453                 {
2454                         .bxt.phy = DPIO_PHY0,
2455                 },
2456         },
2457 };
2458
2459 static const struct i915_power_well_desc glk_power_wells[] = {
2460         {
2461                 .name = "always-on",
2462                 .always_on = 1,
2463                 .domains = POWER_DOMAIN_MASK,
2464                 .ops = &i9xx_always_on_power_well_ops,
2465                 .id = DISP_PW_ID_NONE,
2466         },
2467         {
2468                 .name = "power well 1",
2469                 /* Handled by the DMC firmware */
2470                 .domains = 0,
2471                 .ops = &hsw_power_well_ops,
2472                 .id = SKL_DISP_PW_1,
2473                 {
2474                         .hsw.regs = &hsw_power_well_regs,
2475                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
2476                         .hsw.has_fuses = true,
2477                 },
2478         },
2479         {
2480                 .name = "DC off",
2481                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2482                 .ops = &gen9_dc_off_power_well_ops,
2483                 .id = DISP_PW_ID_NONE,
2484         },
2485         {
2486                 .name = "power well 2",
2487                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2488                 .ops = &hsw_power_well_ops,
2489                 .id = SKL_DISP_PW_2,
2490                 {
2491                         .hsw.regs = &hsw_power_well_regs,
2492                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
2493                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2494                         .hsw.has_vga = true,
2495                         .hsw.has_fuses = true,
2496                 },
2497         },
2498         {
2499                 .name = "dpio-common-a",
2500                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2501                 .ops = &bxt_dpio_cmn_power_well_ops,
2502                 .id = BXT_DISP_PW_DPIO_CMN_A,
2503                 {
2504                         .bxt.phy = DPIO_PHY1,
2505                 },
2506         },
2507         {
2508                 .name = "dpio-common-b",
2509                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2510                 .ops = &bxt_dpio_cmn_power_well_ops,
2511                 .id = VLV_DISP_PW_DPIO_CMN_BC,
2512                 {
2513                         .bxt.phy = DPIO_PHY0,
2514                 },
2515         },
2516         {
2517                 .name = "dpio-common-c",
2518                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2519                 .ops = &bxt_dpio_cmn_power_well_ops,
2520                 .id = GLK_DISP_PW_DPIO_CMN_C,
2521                 {
2522                         .bxt.phy = DPIO_PHY2,
2523                 },
2524         },
2525         {
2526                 .name = "AUX A",
2527                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2528                 .ops = &hsw_power_well_ops,
2529                 .id = DISP_PW_ID_NONE,
2530                 {
2531                         .hsw.regs = &hsw_power_well_regs,
2532                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
2533                 },
2534         },
2535         {
2536                 .name = "AUX B",
2537                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2538                 .ops = &hsw_power_well_ops,
2539                 .id = DISP_PW_ID_NONE,
2540                 {
2541                         .hsw.regs = &hsw_power_well_regs,
2542                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
2543                 },
2544         },
2545         {
2546                 .name = "AUX C",
2547                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2548                 .ops = &hsw_power_well_ops,
2549                 .id = DISP_PW_ID_NONE,
2550                 {
2551                         .hsw.regs = &hsw_power_well_regs,
2552                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
2553                 },
2554         },
2555         {
2556                 .name = "DDI A IO power well",
2557                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2558                 .ops = &hsw_power_well_ops,
2559                 .id = DISP_PW_ID_NONE,
2560                 {
2561                         .hsw.regs = &hsw_power_well_regs,
2562                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
2563                 },
2564         },
2565         {
2566                 .name = "DDI B IO power well",
2567                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2568                 .ops = &hsw_power_well_ops,
2569                 .id = DISP_PW_ID_NONE,
2570                 {
2571                         .hsw.regs = &hsw_power_well_regs,
2572                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
2573                 },
2574         },
2575         {
2576                 .name = "DDI C IO power well",
2577                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2578                 .ops = &hsw_power_well_ops,
2579                 .id = DISP_PW_ID_NONE,
2580                 {
2581                         .hsw.regs = &hsw_power_well_regs,
2582                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
2583                 },
2584         },
2585 };
2586
2587 static const struct i915_power_well_desc cnl_power_wells[] = {
2588         {
2589                 .name = "always-on",
2590                 .always_on = 1,
2591                 .domains = POWER_DOMAIN_MASK,
2592                 .ops = &i9xx_always_on_power_well_ops,
2593                 .id = DISP_PW_ID_NONE,
2594         },
2595         {
2596                 .name = "power well 1",
2597                 /* Handled by the DMC firmware */
2598                 .domains = 0,
2599                 .ops = &hsw_power_well_ops,
2600                 .id = SKL_DISP_PW_1,
2601                 {
2602                         .hsw.regs = &hsw_power_well_regs,
2603                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
2604                         .hsw.has_fuses = true,
2605                 },
2606         },
2607         {
2608                 .name = "AUX A",
2609                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2610                 .ops = &hsw_power_well_ops,
2611                 .id = DISP_PW_ID_NONE,
2612                 {
2613                         .hsw.regs = &hsw_power_well_regs,
2614                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
2615                 },
2616         },
2617         {
2618                 .name = "AUX B",
2619                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2620                 .ops = &hsw_power_well_ops,
2621                 .id = DISP_PW_ID_NONE,
2622                 {
2623                         .hsw.regs = &hsw_power_well_regs,
2624                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
2625                 },
2626         },
2627         {
2628                 .name = "AUX C",
2629                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2630                 .ops = &hsw_power_well_ops,
2631                 .id = DISP_PW_ID_NONE,
2632                 {
2633                         .hsw.regs = &hsw_power_well_regs,
2634                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
2635                 },
2636         },
2637         {
2638                 .name = "AUX D",
2639                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2640                 .ops = &hsw_power_well_ops,
2641                 .id = DISP_PW_ID_NONE,
2642                 {
2643                         .hsw.regs = &hsw_power_well_regs,
2644                         .hsw.idx = CNL_PW_CTL_IDX_AUX_D,
2645                 },
2646         },
2647         {
2648                 .name = "DC off",
2649                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2650                 .ops = &gen9_dc_off_power_well_ops,
2651                 .id = DISP_PW_ID_NONE,
2652         },
2653         {
2654                 .name = "power well 2",
2655                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2656                 .ops = &hsw_power_well_ops,
2657                 .id = SKL_DISP_PW_2,
2658                 {
2659                         .hsw.regs = &hsw_power_well_regs,
2660                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
2661                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2662                         .hsw.has_vga = true,
2663                         .hsw.has_fuses = true,
2664                 },
2665         },
2666         {
2667                 .name = "DDI A IO power well",
2668                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2669                 .ops = &hsw_power_well_ops,
2670                 .id = DISP_PW_ID_NONE,
2671                 {
2672                         .hsw.regs = &hsw_power_well_regs,
2673                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
2674                 },
2675         },
2676         {
2677                 .name = "DDI B IO power well",
2678                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2679                 .ops = &hsw_power_well_ops,
2680                 .id = DISP_PW_ID_NONE,
2681                 {
2682                         .hsw.regs = &hsw_power_well_regs,
2683                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
2684                 },
2685         },
2686         {
2687                 .name = "DDI C IO power well",
2688                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2689                 .ops = &hsw_power_well_ops,
2690                 .id = DISP_PW_ID_NONE,
2691                 {
2692                         .hsw.regs = &hsw_power_well_regs,
2693                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
2694                 },
2695         },
2696         {
2697                 .name = "DDI D IO power well",
2698                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2699                 .ops = &hsw_power_well_ops,
2700                 .id = DISP_PW_ID_NONE,
2701                 {
2702                         .hsw.regs = &hsw_power_well_regs,
2703                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
2704                 },
2705         },
2706         {
2707                 .name = "DDI F IO power well",
2708                 .domains = CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS,
2709                 .ops = &hsw_power_well_ops,
2710                 .id = DISP_PW_ID_NONE,
2711                 {
2712                         .hsw.regs = &hsw_power_well_regs,
2713                         .hsw.idx = CNL_PW_CTL_IDX_DDI_F,
2714                 },
2715         },
2716         {
2717                 .name = "AUX F",
2718                 .domains = CNL_DISPLAY_AUX_F_POWER_DOMAINS,
2719                 .ops = &hsw_power_well_ops,
2720                 .id = DISP_PW_ID_NONE,
2721                 {
2722                         .hsw.regs = &hsw_power_well_regs,
2723                         .hsw.idx = CNL_PW_CTL_IDX_AUX_F,
2724                 },
2725         },
2726 };
2727
2728 static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = {
2729         .sync_hw = hsw_power_well_sync_hw,
2730         .enable = icl_combo_phy_aux_power_well_enable,
2731         .disable = icl_combo_phy_aux_power_well_disable,
2732         .is_enabled = hsw_power_well_enabled,
2733 };
2734
2735 static const struct i915_power_well_regs icl_aux_power_well_regs = {
2736         .bios   = ICL_PWR_WELL_CTL_AUX1,
2737         .driver = ICL_PWR_WELL_CTL_AUX2,
2738         .debug  = ICL_PWR_WELL_CTL_AUX4,
2739 };
2740
2741 static const struct i915_power_well_regs icl_ddi_power_well_regs = {
2742         .bios   = ICL_PWR_WELL_CTL_DDI1,
2743         .driver = ICL_PWR_WELL_CTL_DDI2,
2744         .debug  = ICL_PWR_WELL_CTL_DDI4,
2745 };
2746
2747 static const struct i915_power_well_desc icl_power_wells[] = {
2748         {
2749                 .name = "always-on",
2750                 .always_on = 1,
2751                 .domains = POWER_DOMAIN_MASK,
2752                 .ops = &i9xx_always_on_power_well_ops,
2753                 .id = DISP_PW_ID_NONE,
2754         },
2755         {
2756                 .name = "power well 1",
2757                 /* Handled by the DMC firmware */
2758                 .domains = 0,
2759                 .ops = &hsw_power_well_ops,
2760                 .id = SKL_DISP_PW_1,
2761                 {
2762                         .hsw.regs = &hsw_power_well_regs,
2763                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
2764                         .hsw.has_fuses = true,
2765                 },
2766         },
2767         {
2768                 .name = "power well 2",
2769                 .domains = ICL_PW_2_POWER_DOMAINS,
2770                 .ops = &hsw_power_well_ops,
2771                 .id = SKL_DISP_PW_2,
2772                 {
2773                         .hsw.regs = &hsw_power_well_regs,
2774                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
2775                         .hsw.has_fuses = true,
2776                 },
2777         },
2778         {
2779                 .name = "DC off",
2780                 .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
2781                 .ops = &gen9_dc_off_power_well_ops,
2782                 .id = DISP_PW_ID_NONE,
2783         },
2784         {
2785                 .name = "power well 3",
2786                 .domains = ICL_PW_3_POWER_DOMAINS,
2787                 .ops = &hsw_power_well_ops,
2788                 .id = DISP_PW_ID_NONE,
2789                 {
2790                         .hsw.regs = &hsw_power_well_regs,
2791                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
2792                         .hsw.irq_pipe_mask = BIT(PIPE_B),
2793                         .hsw.has_vga = true,
2794                         .hsw.has_fuses = true,
2795                 },
2796         },
2797         {
2798                 .name = "DDI A IO",
2799                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
2800                 .ops = &hsw_power_well_ops,
2801                 .id = DISP_PW_ID_NONE,
2802                 {
2803                         .hsw.regs = &icl_ddi_power_well_regs,
2804                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
2805                 },
2806         },
2807         {
2808                 .name = "DDI B IO",
2809                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
2810                 .ops = &hsw_power_well_ops,
2811                 .id = DISP_PW_ID_NONE,
2812                 {
2813                         .hsw.regs = &icl_ddi_power_well_regs,
2814                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
2815                 },
2816         },
2817         {
2818                 .name = "DDI C IO",
2819                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
2820                 .ops = &hsw_power_well_ops,
2821                 .id = DISP_PW_ID_NONE,
2822                 {
2823                         .hsw.regs = &icl_ddi_power_well_regs,
2824                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
2825                 },
2826         },
2827         {
2828                 .name = "DDI D IO",
2829                 .domains = ICL_DDI_IO_D_POWER_DOMAINS,
2830                 .ops = &hsw_power_well_ops,
2831                 .id = DISP_PW_ID_NONE,
2832                 {
2833                         .hsw.regs = &icl_ddi_power_well_regs,
2834                         .hsw.idx = ICL_PW_CTL_IDX_DDI_D,
2835                 },
2836         },
2837         {
2838                 .name = "DDI E IO",
2839                 .domains = ICL_DDI_IO_E_POWER_DOMAINS,
2840                 .ops = &hsw_power_well_ops,
2841                 .id = DISP_PW_ID_NONE,
2842                 {
2843                         .hsw.regs = &icl_ddi_power_well_regs,
2844                         .hsw.idx = ICL_PW_CTL_IDX_DDI_E,
2845                 },
2846         },
2847         {
2848                 .name = "DDI F IO",
2849                 .domains = ICL_DDI_IO_F_POWER_DOMAINS,
2850                 .ops = &hsw_power_well_ops,
2851                 .id = DISP_PW_ID_NONE,
2852                 {
2853                         .hsw.regs = &icl_ddi_power_well_regs,
2854                         .hsw.idx = ICL_PW_CTL_IDX_DDI_F,
2855                 },
2856         },
2857         {
2858                 .name = "AUX A",
2859                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
2860                 .ops = &icl_combo_phy_aux_power_well_ops,
2861                 .id = DISP_PW_ID_NONE,
2862                 {
2863                         .hsw.regs = &icl_aux_power_well_regs,
2864                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
2865                 },
2866         },
2867         {
2868                 .name = "AUX B",
2869                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
2870                 .ops = &icl_combo_phy_aux_power_well_ops,
2871                 .id = DISP_PW_ID_NONE,
2872                 {
2873                         .hsw.regs = &icl_aux_power_well_regs,
2874                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
2875                 },
2876         },
2877         {
2878                 .name = "AUX C",
2879                 .domains = ICL_AUX_C_IO_POWER_DOMAINS,
2880                 .ops = &hsw_power_well_ops,
2881                 .id = DISP_PW_ID_NONE,
2882                 {
2883                         .hsw.regs = &icl_aux_power_well_regs,
2884                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
2885                 },
2886         },
2887         {
2888                 .name = "AUX D",
2889                 .domains = ICL_AUX_D_IO_POWER_DOMAINS,
2890                 .ops = &hsw_power_well_ops,
2891                 .id = DISP_PW_ID_NONE,
2892                 {
2893                         .hsw.regs = &icl_aux_power_well_regs,
2894                         .hsw.idx = ICL_PW_CTL_IDX_AUX_D,
2895                 },
2896         },
2897         {
2898                 .name = "AUX E",
2899                 .domains = ICL_AUX_E_IO_POWER_DOMAINS,
2900                 .ops = &hsw_power_well_ops,
2901                 .id = DISP_PW_ID_NONE,
2902                 {
2903                         .hsw.regs = &icl_aux_power_well_regs,
2904                         .hsw.idx = ICL_PW_CTL_IDX_AUX_E,
2905                 },
2906         },
2907         {
2908                 .name = "AUX F",
2909                 .domains = ICL_AUX_F_IO_POWER_DOMAINS,
2910                 .ops = &hsw_power_well_ops,
2911                 .id = DISP_PW_ID_NONE,
2912                 {
2913                         .hsw.regs = &icl_aux_power_well_regs,
2914                         .hsw.idx = ICL_PW_CTL_IDX_AUX_F,
2915                 },
2916         },
2917         {
2918                 .name = "AUX TBT1",
2919                 .domains = ICL_AUX_TBT1_IO_POWER_DOMAINS,
2920                 .ops = &hsw_power_well_ops,
2921                 .id = DISP_PW_ID_NONE,
2922                 {
2923                         .hsw.regs = &icl_aux_power_well_regs,
2924                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT1,
2925                 },
2926         },
2927         {
2928                 .name = "AUX TBT2",
2929                 .domains = ICL_AUX_TBT2_IO_POWER_DOMAINS,
2930                 .ops = &hsw_power_well_ops,
2931                 .id = DISP_PW_ID_NONE,
2932                 {
2933                         .hsw.regs = &icl_aux_power_well_regs,
2934                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT2,
2935                 },
2936         },
2937         {
2938                 .name = "AUX TBT3",
2939                 .domains = ICL_AUX_TBT3_IO_POWER_DOMAINS,
2940                 .ops = &hsw_power_well_ops,
2941                 .id = DISP_PW_ID_NONE,
2942                 {
2943                         .hsw.regs = &icl_aux_power_well_regs,
2944                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT3,
2945                 },
2946         },
2947         {
2948                 .name = "AUX TBT4",
2949                 .domains = ICL_AUX_TBT4_IO_POWER_DOMAINS,
2950                 .ops = &hsw_power_well_ops,
2951                 .id = DISP_PW_ID_NONE,
2952                 {
2953                         .hsw.regs = &icl_aux_power_well_regs,
2954                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT4,
2955                 },
2956         },
2957         {
2958                 .name = "power well 4",
2959                 .domains = ICL_PW_4_POWER_DOMAINS,
2960                 .ops = &hsw_power_well_ops,
2961                 .id = DISP_PW_ID_NONE,
2962                 {
2963                         .hsw.regs = &hsw_power_well_regs,
2964                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
2965                         .hsw.has_fuses = true,
2966                         .hsw.irq_pipe_mask = BIT(PIPE_C),
2967                 },
2968         },
2969 };
2970
2971 static int
2972 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2973                                    int disable_power_well)
2974 {
2975         if (disable_power_well >= 0)
2976                 return !!disable_power_well;
2977
2978         return 1;
2979 }
2980
2981 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2982                                     int enable_dc)
2983 {
2984         uint32_t mask;
2985         int requested_dc;
2986         int max_dc;
2987
2988         if (INTEL_GEN(dev_priv) >= 11) {
2989                 max_dc = 2;
2990                 /*
2991                  * DC9 has a separate HW flow from the rest of the DC states,
2992                  * not depending on the DMC firmware. It's needed by system
2993                  * suspend/resume, so allow it unconditionally.
2994                  */
2995                 mask = DC_STATE_EN_DC9;
2996         } else if (IS_GEN10(dev_priv) || IS_GEN9_BC(dev_priv)) {
2997                 max_dc = 2;
2998                 mask = 0;
2999         } else if (IS_GEN9_LP(dev_priv)) {
3000                 max_dc = 1;
3001                 mask = DC_STATE_EN_DC9;
3002         } else {
3003                 max_dc = 0;
3004                 mask = 0;
3005         }
3006
3007         if (!i915_modparams.disable_power_well)
3008                 max_dc = 0;
3009
3010         if (enable_dc >= 0 && enable_dc <= max_dc) {
3011                 requested_dc = enable_dc;
3012         } else if (enable_dc == -1) {
3013                 requested_dc = max_dc;
3014         } else if (enable_dc > max_dc && enable_dc <= 2) {
3015                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
3016                               enable_dc, max_dc);
3017                 requested_dc = max_dc;
3018         } else {
3019                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
3020                 requested_dc = max_dc;
3021         }
3022
3023         if (requested_dc > 1)
3024                 mask |= DC_STATE_EN_UPTO_DC6;
3025         if (requested_dc > 0)
3026                 mask |= DC_STATE_EN_UPTO_DC5;
3027
3028         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
3029
3030         return mask;
3031 }
3032
3033 static int
3034 __set_power_wells(struct i915_power_domains *power_domains,
3035                   const struct i915_power_well_desc *power_well_descs,
3036                   int power_well_count)
3037 {
3038         u64 power_well_ids = 0;
3039         int i;
3040
3041         power_domains->power_well_count = power_well_count;
3042         power_domains->power_wells =
3043                                 kcalloc(power_well_count,
3044                                         sizeof(*power_domains->power_wells),
3045                                         GFP_KERNEL);
3046         if (!power_domains->power_wells)
3047                 return -ENOMEM;
3048
3049         for (i = 0; i < power_well_count; i++) {
3050                 enum i915_power_well_id id = power_well_descs[i].id;
3051
3052                 power_domains->power_wells[i].desc = &power_well_descs[i];
3053
3054                 if (id == DISP_PW_ID_NONE)
3055                         continue;
3056
3057                 WARN_ON(id >= sizeof(power_well_ids) * 8);
3058                 WARN_ON(power_well_ids & BIT_ULL(id));
3059                 power_well_ids |= BIT_ULL(id);
3060         }
3061
3062         return 0;
3063 }
3064
3065 #define set_power_wells(power_domains, __power_well_descs) \
3066         __set_power_wells(power_domains, __power_well_descs, \
3067                           ARRAY_SIZE(__power_well_descs))
3068
3069 /**
3070  * intel_power_domains_init - initializes the power domain structures
3071  * @dev_priv: i915 device instance
3072  *
3073  * Initializes the power domain structures for @dev_priv depending upon the
3074  * supported platform.
3075  */
3076 int intel_power_domains_init(struct drm_i915_private *dev_priv)
3077 {
3078         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3079         int err;
3080
3081         i915_modparams.disable_power_well =
3082                 sanitize_disable_power_well_option(dev_priv,
3083                                                    i915_modparams.disable_power_well);
3084         dev_priv->csr.allowed_dc_mask =
3085                 get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
3086
3087         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
3088
3089         mutex_init(&power_domains->lock);
3090
3091         /*
3092          * The enabling order will be from lower to higher indexed wells,
3093          * the disabling order is reversed.
3094          */
3095         if (IS_ICELAKE(dev_priv)) {
3096                 err = set_power_wells(power_domains, icl_power_wells);
3097         } else if (IS_CANNONLAKE(dev_priv)) {
3098                 err = set_power_wells(power_domains, cnl_power_wells);
3099
3100                 /*
3101                  * DDI and Aux IO are getting enabled for all ports
3102                  * regardless the presence or use. So, in order to avoid
3103                  * timeouts, lets remove them from the list
3104                  * for the SKUs without port F.
3105                  */
3106                 if (!IS_CNL_WITH_PORT_F(dev_priv))
3107                         power_domains->power_well_count -= 2;
3108         } else if (IS_GEMINILAKE(dev_priv)) {
3109                 err = set_power_wells(power_domains, glk_power_wells);
3110         } else if (IS_BROXTON(dev_priv)) {
3111                 err = set_power_wells(power_domains, bxt_power_wells);
3112         } else if (IS_GEN9_BC(dev_priv)) {
3113                 err = set_power_wells(power_domains, skl_power_wells);
3114         } else if (IS_CHERRYVIEW(dev_priv)) {
3115                 err = set_power_wells(power_domains, chv_power_wells);
3116         } else if (IS_BROADWELL(dev_priv)) {
3117                 err = set_power_wells(power_domains, bdw_power_wells);
3118         } else if (IS_HASWELL(dev_priv)) {
3119                 err = set_power_wells(power_domains, hsw_power_wells);
3120         } else if (IS_VALLEYVIEW(dev_priv)) {
3121                 err = set_power_wells(power_domains, vlv_power_wells);
3122         } else if (IS_I830(dev_priv)) {
3123                 err = set_power_wells(power_domains, i830_power_wells);
3124         } else {
3125                 err = set_power_wells(power_domains, i9xx_always_on_power_well);
3126         }
3127
3128         return err;
3129 }
3130
3131 /**
3132  * intel_power_domains_cleanup - clean up power domains resources
3133  * @dev_priv: i915 device instance
3134  *
3135  * Release any resources acquired by intel_power_domains_init()
3136  */
3137 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
3138 {
3139         kfree(dev_priv->power_domains.power_wells);
3140 }
3141
3142 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
3143 {
3144         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3145         struct i915_power_well *power_well;
3146
3147         mutex_lock(&power_domains->lock);
3148         for_each_power_well(dev_priv, power_well) {
3149                 power_well->desc->ops->sync_hw(dev_priv, power_well);
3150                 power_well->hw_enabled =
3151                         power_well->desc->ops->is_enabled(dev_priv, power_well);
3152         }
3153         mutex_unlock(&power_domains->lock);
3154 }
3155
3156 static inline
3157 bool intel_dbuf_slice_set(struct drm_i915_private *dev_priv,
3158                           i915_reg_t reg, bool enable)
3159 {
3160         u32 val, status;
3161
3162         val = I915_READ(reg);
3163         val = enable ? (val | DBUF_POWER_REQUEST) : (val & ~DBUF_POWER_REQUEST);
3164         I915_WRITE(reg, val);
3165         POSTING_READ(reg);
3166         udelay(10);
3167
3168         status = I915_READ(reg) & DBUF_POWER_STATE;
3169         if ((enable && !status) || (!enable && status)) {
3170                 DRM_ERROR("DBus power %s timeout!\n",
3171                           enable ? "enable" : "disable");
3172                 return false;
3173         }
3174         return true;
3175 }
3176
3177 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
3178 {
3179         intel_dbuf_slice_set(dev_priv, DBUF_CTL, true);
3180 }
3181
3182 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
3183 {
3184         intel_dbuf_slice_set(dev_priv, DBUF_CTL, false);
3185 }
3186
3187 static u8 intel_dbuf_max_slices(struct drm_i915_private *dev_priv)
3188 {
3189         if (INTEL_GEN(dev_priv) < 11)
3190                 return 1;
3191         return 2;
3192 }
3193
3194 void icl_dbuf_slices_update(struct drm_i915_private *dev_priv,
3195                             u8 req_slices)
3196 {
3197         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
3198         u32 val;
3199         bool ret;
3200
3201         if (req_slices > intel_dbuf_max_slices(dev_priv)) {
3202                 DRM_ERROR("Invalid number of dbuf slices requested\n");
3203                 return;
3204         }
3205
3206         if (req_slices == hw_enabled_slices || req_slices == 0)
3207                 return;
3208
3209         val = I915_READ(DBUF_CTL_S2);
3210         if (req_slices > hw_enabled_slices)
3211                 ret = intel_dbuf_slice_set(dev_priv, DBUF_CTL_S2, true);
3212         else
3213                 ret = intel_dbuf_slice_set(dev_priv, DBUF_CTL_S2, false);
3214
3215         if (ret)
3216                 dev_priv->wm.skl_hw.ddb.enabled_slices = req_slices;
3217 }
3218
3219 static void icl_dbuf_enable(struct drm_i915_private *dev_priv)
3220 {
3221         I915_WRITE(DBUF_CTL_S1, I915_READ(DBUF_CTL_S1) | DBUF_POWER_REQUEST);
3222         I915_WRITE(DBUF_CTL_S2, I915_READ(DBUF_CTL_S2) | DBUF_POWER_REQUEST);
3223         POSTING_READ(DBUF_CTL_S2);
3224
3225         udelay(10);
3226
3227         if (!(I915_READ(DBUF_CTL_S1) & DBUF_POWER_STATE) ||
3228             !(I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
3229                 DRM_ERROR("DBuf power enable timeout\n");
3230         else
3231                 dev_priv->wm.skl_hw.ddb.enabled_slices = 2;
3232 }
3233
3234 static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
3235 {
3236         I915_WRITE(DBUF_CTL_S1, I915_READ(DBUF_CTL_S1) & ~DBUF_POWER_REQUEST);
3237         I915_WRITE(DBUF_CTL_S2, I915_READ(DBUF_CTL_S2) & ~DBUF_POWER_REQUEST);
3238         POSTING_READ(DBUF_CTL_S2);
3239
3240         udelay(10);
3241
3242         if ((I915_READ(DBUF_CTL_S1) & DBUF_POWER_STATE) ||
3243             (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
3244                 DRM_ERROR("DBuf power disable timeout!\n");
3245         else
3246                 dev_priv->wm.skl_hw.ddb.enabled_slices = 0;
3247 }
3248
3249 static void icl_mbus_init(struct drm_i915_private *dev_priv)
3250 {
3251         uint32_t val;
3252
3253         val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
3254               MBUS_ABOX_BT_CREDIT_POOL2(16) |
3255               MBUS_ABOX_B_CREDIT(1) |
3256               MBUS_ABOX_BW_CREDIT(1);
3257
3258         I915_WRITE(MBUS_ABOX_CTL, val);
3259 }
3260
3261 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
3262                                       bool enable)
3263 {
3264         i915_reg_t reg;
3265         u32 reset_bits, val;
3266
3267         if (IS_IVYBRIDGE(dev_priv)) {
3268                 reg = GEN7_MSG_CTL;
3269                 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
3270         } else {
3271                 reg = HSW_NDE_RSTWRN_OPT;
3272                 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
3273         }
3274
3275         val = I915_READ(reg);
3276
3277         if (enable)
3278                 val |= reset_bits;
3279         else
3280                 val &= ~reset_bits;
3281
3282         I915_WRITE(reg, val);
3283 }
3284
3285 static void skl_display_core_init(struct drm_i915_private *dev_priv,
3286                                    bool resume)
3287 {
3288         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3289         struct i915_power_well *well;
3290
3291         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3292
3293         /* enable PCH reset handshake */
3294         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
3295
3296         /* enable PG1 and Misc I/O */
3297         mutex_lock(&power_domains->lock);
3298
3299         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3300         intel_power_well_enable(dev_priv, well);
3301
3302         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
3303         intel_power_well_enable(dev_priv, well);
3304
3305         mutex_unlock(&power_domains->lock);
3306
3307         skl_init_cdclk(dev_priv);
3308
3309         gen9_dbuf_enable(dev_priv);
3310
3311         if (resume && dev_priv->csr.dmc_payload)
3312                 intel_csr_load_program(dev_priv);
3313 }
3314
3315 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
3316 {
3317         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3318         struct i915_power_well *well;
3319
3320         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3321
3322         gen9_dbuf_disable(dev_priv);
3323
3324         skl_uninit_cdclk(dev_priv);
3325
3326         /* The spec doesn't call for removing the reset handshake flag */
3327         /* disable PG1 and Misc I/O */
3328
3329         mutex_lock(&power_domains->lock);
3330
3331         /*
3332          * BSpec says to keep the MISC IO power well enabled here, only
3333          * remove our request for power well 1.
3334          * Note that even though the driver's request is removed power well 1
3335          * may stay enabled after this due to DMC's own request on it.
3336          */
3337         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3338         intel_power_well_disable(dev_priv, well);
3339
3340         mutex_unlock(&power_domains->lock);
3341
3342         usleep_range(10, 30);           /* 10 us delay per Bspec */
3343 }
3344
3345 void bxt_display_core_init(struct drm_i915_private *dev_priv,
3346                            bool resume)
3347 {
3348         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3349         struct i915_power_well *well;
3350
3351         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3352
3353         /*
3354          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
3355          * or else the reset will hang because there is no PCH to respond.
3356          * Move the handshake programming to initialization sequence.
3357          * Previously was left up to BIOS.
3358          */
3359         intel_pch_reset_handshake(dev_priv, false);
3360
3361         /* Enable PG1 */
3362         mutex_lock(&power_domains->lock);
3363
3364         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3365         intel_power_well_enable(dev_priv, well);
3366
3367         mutex_unlock(&power_domains->lock);
3368
3369         bxt_init_cdclk(dev_priv);
3370
3371         gen9_dbuf_enable(dev_priv);
3372
3373         if (resume && dev_priv->csr.dmc_payload)
3374                 intel_csr_load_program(dev_priv);
3375 }
3376
3377 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
3378 {
3379         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3380         struct i915_power_well *well;
3381
3382         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3383
3384         gen9_dbuf_disable(dev_priv);
3385
3386         bxt_uninit_cdclk(dev_priv);
3387
3388         /* The spec doesn't call for removing the reset handshake flag */
3389
3390         /*
3391          * Disable PW1 (PG1).
3392          * Note that even though the driver's request is removed power well 1
3393          * may stay enabled after this due to DMC's own request on it.
3394          */
3395         mutex_lock(&power_domains->lock);
3396
3397         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3398         intel_power_well_disable(dev_priv, well);
3399
3400         mutex_unlock(&power_domains->lock);
3401
3402         usleep_range(10, 30);           /* 10 us delay per Bspec */
3403 }
3404
3405 enum {
3406         PROCMON_0_85V_DOT_0,
3407         PROCMON_0_95V_DOT_0,
3408         PROCMON_0_95V_DOT_1,
3409         PROCMON_1_05V_DOT_0,
3410         PROCMON_1_05V_DOT_1,
3411 };
3412
3413 static const struct cnl_procmon {
3414         u32 dw1, dw9, dw10;
3415 } cnl_procmon_values[] = {
3416         [PROCMON_0_85V_DOT_0] =
3417                 { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
3418         [PROCMON_0_95V_DOT_0] =
3419                 { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
3420         [PROCMON_0_95V_DOT_1] =
3421                 { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
3422         [PROCMON_1_05V_DOT_0] =
3423                 { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
3424         [PROCMON_1_05V_DOT_1] =
3425                 { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
3426 };
3427
3428 /*
3429  * CNL has just one set of registers, while ICL has two sets: one for port A and
3430  * the other for port B. The CNL registers are equivalent to the ICL port A
3431  * registers, that's why we call the ICL macros even though the function has CNL
3432  * on its name.
3433  */
3434 static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv,
3435                                        enum port port)
3436 {
3437         const struct cnl_procmon *procmon;
3438         u32 val;
3439
3440         val = I915_READ(ICL_PORT_COMP_DW3(port));
3441         switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
3442         default:
3443                 MISSING_CASE(val);
3444                 /* fall through */
3445         case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
3446                 procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
3447                 break;
3448         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0:
3449                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0];
3450                 break;
3451         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1:
3452                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1];
3453                 break;
3454         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0:
3455                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0];
3456                 break;
3457         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1:
3458                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1];
3459                 break;
3460         }
3461
3462         val = I915_READ(ICL_PORT_COMP_DW1(port));
3463         val &= ~((0xff << 16) | 0xff);
3464         val |= procmon->dw1;
3465         I915_WRITE(ICL_PORT_COMP_DW1(port), val);
3466
3467         I915_WRITE(ICL_PORT_COMP_DW9(port), procmon->dw9);
3468         I915_WRITE(ICL_PORT_COMP_DW10(port), procmon->dw10);
3469 }
3470
3471 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
3472 {
3473         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3474         struct i915_power_well *well;
3475         u32 val;
3476
3477         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3478
3479         /* 1. Enable PCH Reset Handshake */
3480         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
3481
3482         /* 2. Enable Comp */
3483         val = I915_READ(CHICKEN_MISC_2);
3484         val &= ~CNL_COMP_PWR_DOWN;
3485         I915_WRITE(CHICKEN_MISC_2, val);
3486
3487         /* Dummy PORT_A to get the correct CNL register from the ICL macro */
3488         cnl_set_procmon_ref_values(dev_priv, PORT_A);
3489
3490         val = I915_READ(CNL_PORT_COMP_DW0);
3491         val |= COMP_INIT;
3492         I915_WRITE(CNL_PORT_COMP_DW0, val);
3493
3494         /* 3. */
3495         val = I915_READ(CNL_PORT_CL1CM_DW5);
3496         val |= CL_POWER_DOWN_ENABLE;
3497         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
3498
3499         /*
3500          * 4. Enable Power Well 1 (PG1).
3501          *    The AUX IO power wells will be enabled on demand.
3502          */
3503         mutex_lock(&power_domains->lock);
3504         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3505         intel_power_well_enable(dev_priv, well);
3506         mutex_unlock(&power_domains->lock);
3507
3508         /* 5. Enable CD clock */
3509         cnl_init_cdclk(dev_priv);
3510
3511         /* 6. Enable DBUF */
3512         gen9_dbuf_enable(dev_priv);
3513
3514         if (resume && dev_priv->csr.dmc_payload)
3515                 intel_csr_load_program(dev_priv);
3516 }
3517
3518 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
3519 {
3520         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3521         struct i915_power_well *well;
3522         u32 val;
3523
3524         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3525
3526         /* 1. Disable all display engine functions -> aready done */
3527
3528         /* 2. Disable DBUF */
3529         gen9_dbuf_disable(dev_priv);
3530
3531         /* 3. Disable CD clock */
3532         cnl_uninit_cdclk(dev_priv);
3533
3534         /*
3535          * 4. Disable Power Well 1 (PG1).
3536          *    The AUX IO power wells are toggled on demand, so they are already
3537          *    disabled at this point.
3538          */
3539         mutex_lock(&power_domains->lock);
3540         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3541         intel_power_well_disable(dev_priv, well);
3542         mutex_unlock(&power_domains->lock);
3543
3544         usleep_range(10, 30);           /* 10 us delay per Bspec */
3545
3546         /* 5. Disable Comp */
3547         val = I915_READ(CHICKEN_MISC_2);
3548         val |= CNL_COMP_PWR_DOWN;
3549         I915_WRITE(CHICKEN_MISC_2, val);
3550 }
3551
3552 void icl_display_core_init(struct drm_i915_private *dev_priv,
3553                            bool resume)
3554 {
3555         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3556         struct i915_power_well *well;
3557         enum port port;
3558         u32 val;
3559
3560         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3561
3562         /* 1. Enable PCH reset handshake. */
3563         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
3564
3565         for (port = PORT_A; port <= PORT_B; port++) {
3566                 /* 2. Enable DDI combo PHY comp. */
3567                 val = I915_READ(ICL_PHY_MISC(port));
3568                 val &= ~ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN;
3569                 I915_WRITE(ICL_PHY_MISC(port), val);
3570
3571                 cnl_set_procmon_ref_values(dev_priv, port);
3572
3573                 val = I915_READ(ICL_PORT_COMP_DW0(port));
3574                 val |= COMP_INIT;
3575                 I915_WRITE(ICL_PORT_COMP_DW0(port), val);
3576
3577                 /* 3. Set power down enable. */
3578                 val = I915_READ(ICL_PORT_CL_DW5(port));
3579                 val |= CL_POWER_DOWN_ENABLE;
3580                 I915_WRITE(ICL_PORT_CL_DW5(port), val);
3581         }
3582
3583         /*
3584          * 4. Enable Power Well 1 (PG1).
3585          *    The AUX IO power wells will be enabled on demand.
3586          */
3587         mutex_lock(&power_domains->lock);
3588         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3589         intel_power_well_enable(dev_priv, well);
3590         mutex_unlock(&power_domains->lock);
3591
3592         /* 5. Enable CDCLK. */
3593         icl_init_cdclk(dev_priv);
3594
3595         /* 6. Enable DBUF. */
3596         icl_dbuf_enable(dev_priv);
3597
3598         /* 7. Setup MBUS. */
3599         icl_mbus_init(dev_priv);
3600
3601         if (resume && dev_priv->csr.dmc_payload)
3602                 intel_csr_load_program(dev_priv);
3603 }
3604
3605 void icl_display_core_uninit(struct drm_i915_private *dev_priv)
3606 {
3607         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3608         struct i915_power_well *well;
3609         enum port port;
3610         u32 val;
3611
3612         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
3613
3614         /* 1. Disable all display engine functions -> aready done */
3615
3616         /* 2. Disable DBUF */
3617         icl_dbuf_disable(dev_priv);
3618
3619         /* 3. Disable CD clock */
3620         icl_uninit_cdclk(dev_priv);
3621
3622         /*
3623          * 4. Disable Power Well 1 (PG1).
3624          *    The AUX IO power wells are toggled on demand, so they are already
3625          *    disabled at this point.
3626          */
3627         mutex_lock(&power_domains->lock);
3628         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
3629         intel_power_well_disable(dev_priv, well);
3630         mutex_unlock(&power_domains->lock);
3631
3632         /* 5. Disable Comp */
3633         for (port = PORT_A; port <= PORT_B; port++) {
3634                 val = I915_READ(ICL_PHY_MISC(port));
3635                 val |= ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN;
3636                 I915_WRITE(ICL_PHY_MISC(port), val);
3637         }
3638 }
3639
3640 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
3641 {
3642         struct i915_power_well *cmn_bc =
3643                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
3644         struct i915_power_well *cmn_d =
3645                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
3646
3647         /*
3648          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
3649          * workaround never ever read DISPLAY_PHY_CONTROL, and
3650          * instead maintain a shadow copy ourselves. Use the actual
3651          * power well state and lane status to reconstruct the
3652          * expected initial value.
3653          */
3654         dev_priv->chv_phy_control =
3655                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
3656                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
3657                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
3658                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
3659                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
3660
3661         /*
3662          * If all lanes are disabled we leave the override disabled
3663          * with all power down bits cleared to match the state we
3664          * would use after disabling the port. Otherwise enable the
3665          * override and set the lane powerdown bits accding to the
3666          * current lane status.
3667          */
3668         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
3669                 uint32_t status = I915_READ(DPLL(PIPE_A));
3670                 unsigned int mask;
3671
3672                 mask = status & DPLL_PORTB_READY_MASK;
3673                 if (mask == 0xf)
3674                         mask = 0x0;
3675                 else
3676                         dev_priv->chv_phy_control |=
3677                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
3678
3679                 dev_priv->chv_phy_control |=
3680                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
3681
3682                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
3683                 if (mask == 0xf)
3684                         mask = 0x0;
3685                 else
3686                         dev_priv->chv_phy_control |=
3687                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
3688
3689                 dev_priv->chv_phy_control |=
3690                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
3691
3692                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
3693
3694                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
3695         } else {
3696                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
3697         }
3698
3699         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
3700                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
3701                 unsigned int mask;
3702
3703                 mask = status & DPLL_PORTD_READY_MASK;
3704
3705                 if (mask == 0xf)
3706                         mask = 0x0;
3707                 else
3708                         dev_priv->chv_phy_control |=
3709                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
3710
3711                 dev_priv->chv_phy_control |=
3712                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
3713
3714                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
3715
3716                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
3717         } else {
3718                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
3719         }
3720
3721         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
3722
3723         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
3724                       dev_priv->chv_phy_control);
3725 }
3726
3727 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
3728 {
3729         struct i915_power_well *cmn =
3730                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
3731         struct i915_power_well *disp2d =
3732                 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
3733
3734         /* If the display might be already active skip this */
3735         if (cmn->desc->ops->is_enabled(dev_priv, cmn) &&
3736             disp2d->desc->ops->is_enabled(dev_priv, disp2d) &&
3737             I915_READ(DPIO_CTL) & DPIO_CMNRST)
3738                 return;
3739
3740         DRM_DEBUG_KMS("toggling display PHY side reset\n");
3741
3742         /* cmnlane needs DPLL registers */
3743         disp2d->desc->ops->enable(dev_priv, disp2d);
3744
3745         /*
3746          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
3747          * Need to assert and de-assert PHY SB reset by gating the
3748          * common lane power, then un-gating it.
3749          * Simply ungating isn't enough to reset the PHY enough to get
3750          * ports and lanes running.
3751          */
3752         cmn->desc->ops->disable(dev_priv, cmn);
3753 }
3754
3755 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
3756
3757 /**
3758  * intel_power_domains_init_hw - initialize hardware power domain state
3759  * @dev_priv: i915 device instance
3760  * @resume: Called from resume code paths or not
3761  *
3762  * This function initializes the hardware power domain state and enables all
3763  * power wells belonging to the INIT power domain. Power wells in other
3764  * domains (and not in the INIT domain) are referenced or disabled by
3765  * intel_modeset_readout_hw_state(). After that the reference count of each
3766  * power well must match its HW enabled state, see
3767  * intel_power_domains_verify_state().
3768  *
3769  * It will return with power domains disabled (to be enabled later by
3770  * intel_power_domains_enable()) and must be paired with
3771  * intel_power_domains_fini_hw().
3772  */
3773 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
3774 {
3775         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3776
3777         power_domains->initializing = true;
3778
3779         if (IS_ICELAKE(dev_priv)) {
3780                 icl_display_core_init(dev_priv, resume);
3781         } else if (IS_CANNONLAKE(dev_priv)) {
3782                 cnl_display_core_init(dev_priv, resume);
3783         } else if (IS_GEN9_BC(dev_priv)) {
3784                 skl_display_core_init(dev_priv, resume);
3785         } else if (IS_GEN9_LP(dev_priv)) {
3786                 bxt_display_core_init(dev_priv, resume);
3787         } else if (IS_CHERRYVIEW(dev_priv)) {
3788                 mutex_lock(&power_domains->lock);
3789                 chv_phy_control_init(dev_priv);
3790                 mutex_unlock(&power_domains->lock);
3791         } else if (IS_VALLEYVIEW(dev_priv)) {
3792                 mutex_lock(&power_domains->lock);
3793                 vlv_cmnlane_wa(dev_priv);
3794                 mutex_unlock(&power_domains->lock);
3795         } else if (IS_IVYBRIDGE(dev_priv) || INTEL_GEN(dev_priv) >= 7)
3796                 intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
3797
3798         /*
3799          * Keep all power wells enabled for any dependent HW access during
3800          * initialization and to make sure we keep BIOS enabled display HW
3801          * resources powered until display HW readout is complete. We drop
3802          * this reference in intel_power_domains_enable().
3803          */
3804         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3805         /* Disable power support if the user asked so. */
3806         if (!i915_modparams.disable_power_well)
3807                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3808         intel_power_domains_sync_hw(dev_priv);
3809
3810         power_domains->initializing = false;
3811 }
3812
3813 /**
3814  * intel_power_domains_fini_hw - deinitialize hw power domain state
3815  * @dev_priv: i915 device instance
3816  *
3817  * De-initializes the display power domain HW state. It also ensures that the
3818  * device stays powered up so that the driver can be reloaded.
3819  *
3820  * It must be called with power domains already disabled (after a call to
3821  * intel_power_domains_disable()) and must be paired with
3822  * intel_power_domains_init_hw().
3823  */
3824 void intel_power_domains_fini_hw(struct drm_i915_private *dev_priv)
3825 {
3826         /* Keep the power well enabled, but cancel its rpm wakeref. */
3827         intel_runtime_pm_put(dev_priv);
3828
3829         /* Remove the refcount we took to keep power well support disabled. */
3830         if (!i915_modparams.disable_power_well)
3831                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3832
3833         intel_power_domains_verify_state(dev_priv);
3834 }
3835
3836 /**
3837  * intel_power_domains_enable - enable toggling of display power wells
3838  * @dev_priv: i915 device instance
3839  *
3840  * Enable the ondemand enabling/disabling of the display power wells. Note that
3841  * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
3842  * only at specific points of the display modeset sequence, thus they are not
3843  * affected by the intel_power_domains_enable()/disable() calls. The purpose
3844  * of these function is to keep the rest of power wells enabled until the end
3845  * of display HW readout (which will acquire the power references reflecting
3846  * the current HW state).
3847  */
3848 void intel_power_domains_enable(struct drm_i915_private *dev_priv)
3849 {
3850         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3851
3852         intel_power_domains_verify_state(dev_priv);
3853 }
3854
3855 /**
3856  * intel_power_domains_disable - disable toggling of display power wells
3857  * @dev_priv: i915 device instance
3858  *
3859  * Disable the ondemand enabling/disabling of the display power wells. See
3860  * intel_power_domains_enable() for which power wells this call controls.
3861  */
3862 void intel_power_domains_disable(struct drm_i915_private *dev_priv)
3863 {
3864         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3865
3866         intel_power_domains_verify_state(dev_priv);
3867 }
3868
3869 /**
3870  * intel_power_domains_suspend - suspend power domain state
3871  * @dev_priv: i915 device instance
3872  * @suspend_mode: specifies the target suspend state (idle, mem, hibernation)
3873  *
3874  * This function prepares the hardware power domain state before entering
3875  * system suspend.
3876  *
3877  * It must be called with power domains already disabled (after a call to
3878  * intel_power_domains_disable()) and paired with intel_power_domains_resume().
3879  */
3880 void intel_power_domains_suspend(struct drm_i915_private *dev_priv,
3881                                  enum i915_drm_suspend_mode suspend_mode)
3882 {
3883         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3884
3885         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3886
3887         /*
3888          * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
3889          * support don't manually deinit the power domains. This also means the
3890          * CSR/DMC firmware will stay active, it will power down any HW
3891          * resources as required and also enable deeper system power states
3892          * that would be blocked if the firmware was inactive.
3893          */
3894         if (!(dev_priv->csr.allowed_dc_mask & DC_STATE_EN_DC9) &&
3895             suspend_mode == I915_DRM_SUSPEND_IDLE &&
3896             dev_priv->csr.dmc_payload != NULL) {
3897                 intel_power_domains_verify_state(dev_priv);
3898                 return;
3899         }
3900
3901         /*
3902          * Even if power well support was disabled we still want to disable
3903          * power wells if power domains must be deinitialized for suspend.
3904          */
3905         if (!i915_modparams.disable_power_well) {
3906                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3907                 intel_power_domains_verify_state(dev_priv);
3908         }
3909
3910         if (IS_ICELAKE(dev_priv))
3911                 icl_display_core_uninit(dev_priv);
3912         else if (IS_CANNONLAKE(dev_priv))
3913                 cnl_display_core_uninit(dev_priv);
3914         else if (IS_GEN9_BC(dev_priv))
3915                 skl_display_core_uninit(dev_priv);
3916         else if (IS_GEN9_LP(dev_priv))
3917                 bxt_display_core_uninit(dev_priv);
3918
3919         power_domains->display_core_suspended = true;
3920 }
3921
3922 /**
3923  * intel_power_domains_resume - resume power domain state
3924  * @dev_priv: i915 device instance
3925  *
3926  * This function resume the hardware power domain state during system resume.
3927  *
3928  * It will return with power domain support disabled (to be enabled later by
3929  * intel_power_domains_enable()) and must be paired with
3930  * intel_power_domains_suspend().
3931  */
3932 void intel_power_domains_resume(struct drm_i915_private *dev_priv)
3933 {
3934         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3935
3936         if (power_domains->display_core_suspended) {
3937                 intel_power_domains_init_hw(dev_priv, true);
3938                 power_domains->display_core_suspended = false;
3939         } else {
3940                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3941         }
3942
3943         intel_power_domains_verify_state(dev_priv);
3944 }
3945
3946 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
3947
3948 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3949 {
3950         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3951         struct i915_power_well *power_well;
3952
3953         for_each_power_well(dev_priv, power_well) {
3954                 enum intel_display_power_domain domain;
3955
3956                 DRM_DEBUG_DRIVER("%-25s %d\n",
3957                                  power_well->desc->name, power_well->count);
3958
3959                 for_each_power_domain(domain, power_well->desc->domains)
3960                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3961                                          intel_display_power_domain_str(domain),
3962                                          power_domains->domain_use_count[domain]);
3963         }
3964 }
3965
3966 /**
3967  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3968  * @dev_priv: i915 device instance
3969  *
3970  * Verify if the reference count of each power well matches its HW enabled
3971  * state and the total refcount of the domains it belongs to. This must be
3972  * called after modeset HW state sanitization, which is responsible for
3973  * acquiring reference counts for any power wells in use and disabling the
3974  * ones left on by BIOS but not required by any active output.
3975  */
3976 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3977 {
3978         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3979         struct i915_power_well *power_well;
3980         bool dump_domain_info;
3981
3982         mutex_lock(&power_domains->lock);
3983
3984         dump_domain_info = false;
3985         for_each_power_well(dev_priv, power_well) {
3986                 enum intel_display_power_domain domain;
3987                 int domains_count;
3988                 bool enabled;
3989
3990                 /*
3991                  * Power wells not belonging to any domain (like the MISC_IO
3992                  * and PW1 power wells) are under FW control, so ignore them,
3993                  * since their state can change asynchronously.
3994                  */
3995                 if (!power_well->desc->domains)
3996                         continue;
3997
3998                 enabled = power_well->desc->ops->is_enabled(dev_priv,
3999                                                             power_well);
4000                 if ((power_well->count || power_well->desc->always_on) !=
4001                     enabled)
4002                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
4003                                   power_well->desc->name,
4004                                   power_well->count, enabled);
4005
4006                 domains_count = 0;
4007                 for_each_power_domain(domain, power_well->desc->domains)
4008                         domains_count += power_domains->domain_use_count[domain];
4009
4010                 if (power_well->count != domains_count) {
4011                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
4012                                   "(refcount %d/domains refcount %d)\n",
4013                                   power_well->desc->name, power_well->count,
4014                                   domains_count);
4015                         dump_domain_info = true;
4016                 }
4017         }
4018
4019         if (dump_domain_info) {
4020                 static bool dumped;
4021
4022                 if (!dumped) {
4023                         intel_power_domains_dump_info(dev_priv);
4024                         dumped = true;
4025                 }
4026         }
4027
4028         mutex_unlock(&power_domains->lock);
4029 }
4030
4031 #else
4032
4033 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
4034 {
4035 }
4036
4037 #endif
4038
4039 /**
4040  * intel_runtime_pm_get - grab a runtime pm reference
4041  * @dev_priv: i915 device instance
4042  *
4043  * This function grabs a device-level runtime pm reference (mostly used for GEM
4044  * code to ensure the GTT or GT is on) and ensures that it is powered up.
4045  *
4046  * Any runtime pm reference obtained by this function must have a symmetric
4047  * call to intel_runtime_pm_put() to release the reference again.
4048  */
4049 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
4050 {
4051         struct pci_dev *pdev = dev_priv->drm.pdev;
4052         struct device *kdev = &pdev->dev;
4053         int ret;
4054
4055         ret = pm_runtime_get_sync(kdev);
4056         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
4057
4058         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
4059         assert_rpm_wakelock_held(dev_priv);
4060 }
4061
4062 /**
4063  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
4064  * @dev_priv: i915 device instance
4065  *
4066  * This function grabs a device-level runtime pm reference if the device is
4067  * already in use and ensures that it is powered up. It is illegal to try
4068  * and access the HW should intel_runtime_pm_get_if_in_use() report failure.
4069  *
4070  * Any runtime pm reference obtained by this function must have a symmetric
4071  * call to intel_runtime_pm_put() to release the reference again.
4072  *
4073  * Returns: True if the wakeref was acquired, or False otherwise.
4074  */
4075 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
4076 {
4077         if (IS_ENABLED(CONFIG_PM)) {
4078                 struct pci_dev *pdev = dev_priv->drm.pdev;
4079                 struct device *kdev = &pdev->dev;
4080
4081                 /*
4082                  * In cases runtime PM is disabled by the RPM core and we get
4083                  * an -EINVAL return value we are not supposed to call this
4084                  * function, since the power state is undefined. This applies
4085                  * atm to the late/early system suspend/resume handlers.
4086                  */
4087                 if (pm_runtime_get_if_in_use(kdev) <= 0)
4088                         return false;
4089         }
4090
4091         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
4092         assert_rpm_wakelock_held(dev_priv);
4093
4094         return true;
4095 }
4096
4097 /**
4098  * intel_runtime_pm_get_noresume - grab a runtime pm reference
4099  * @dev_priv: i915 device instance
4100  *
4101  * This function grabs a device-level runtime pm reference (mostly used for GEM
4102  * code to ensure the GTT or GT is on).
4103  *
4104  * It will _not_ power up the device but instead only check that it's powered
4105  * on.  Therefore it is only valid to call this functions from contexts where
4106  * the device is known to be powered up and where trying to power it up would
4107  * result in hilarity and deadlocks. That pretty much means only the system
4108  * suspend/resume code where this is used to grab runtime pm references for
4109  * delayed setup down in work items.
4110  *
4111  * Any runtime pm reference obtained by this function must have a symmetric
4112  * call to intel_runtime_pm_put() to release the reference again.
4113  */
4114 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
4115 {
4116         struct pci_dev *pdev = dev_priv->drm.pdev;
4117         struct device *kdev = &pdev->dev;
4118
4119         assert_rpm_wakelock_held(dev_priv);
4120         pm_runtime_get_noresume(kdev);
4121
4122         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
4123 }
4124
4125 /**
4126  * intel_runtime_pm_put - release a runtime pm reference
4127  * @dev_priv: i915 device instance
4128  *
4129  * This function drops the device-level runtime pm reference obtained by
4130  * intel_runtime_pm_get() and might power down the corresponding
4131  * hardware block right away if this is the last reference.
4132  */
4133 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
4134 {
4135         struct pci_dev *pdev = dev_priv->drm.pdev;
4136         struct device *kdev = &pdev->dev;
4137
4138         assert_rpm_wakelock_held(dev_priv);
4139         atomic_dec(&dev_priv->runtime_pm.wakeref_count);
4140
4141         pm_runtime_mark_last_busy(kdev);
4142         pm_runtime_put_autosuspend(kdev);
4143 }
4144
4145 /**
4146  * intel_runtime_pm_enable - enable runtime pm
4147  * @dev_priv: i915 device instance
4148  *
4149  * This function enables runtime pm at the end of the driver load sequence.
4150  *
4151  * Note that this function does currently not enable runtime pm for the
4152  * subordinate display power domains. That is done by
4153  * intel_power_domains_enable().
4154  */
4155 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
4156 {
4157         struct pci_dev *pdev = dev_priv->drm.pdev;
4158         struct device *kdev = &pdev->dev;
4159
4160         /*
4161          * Disable the system suspend direct complete optimization, which can
4162          * leave the device suspended skipping the driver's suspend handlers
4163          * if the device was already runtime suspended. This is needed due to
4164          * the difference in our runtime and system suspend sequence and
4165          * becaue the HDA driver may require us to enable the audio power
4166          * domain during system suspend.
4167          */
4168         dev_pm_set_driver_flags(kdev, DPM_FLAG_NEVER_SKIP);
4169
4170         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
4171         pm_runtime_mark_last_busy(kdev);
4172
4173         /*
4174          * Take a permanent reference to disable the RPM functionality and drop
4175          * it only when unloading the driver. Use the low level get/put helpers,
4176          * so the driver's own RPM reference tracking asserts also work on
4177          * platforms without RPM support.
4178          */
4179         if (!HAS_RUNTIME_PM(dev_priv)) {
4180                 int ret;
4181
4182                 pm_runtime_dont_use_autosuspend(kdev);
4183                 ret = pm_runtime_get_sync(kdev);
4184                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
4185         } else {
4186                 pm_runtime_use_autosuspend(kdev);
4187         }
4188
4189         /*
4190          * The core calls the driver load handler with an RPM reference held.
4191          * We drop that here and will reacquire it during unloading in
4192          * intel_power_domains_fini().
4193          */
4194         pm_runtime_put_autosuspend(kdev);
4195 }
4196
4197 void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
4198 {
4199         struct pci_dev *pdev = dev_priv->drm.pdev;
4200         struct device *kdev = &pdev->dev;
4201
4202         /* Transfer rpm ownership back to core */
4203         WARN(pm_runtime_get_sync(&dev_priv->drm.pdev->dev) < 0,
4204              "Failed to pass rpm ownership back to core\n");
4205
4206         pm_runtime_dont_use_autosuspend(kdev);
4207
4208         if (!HAS_RUNTIME_PM(dev_priv))
4209                 pm_runtime_put(kdev);
4210 }