drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include "intel_frontbuffer.h"
  38 #include <drm/i915_drm.h>
  39 #include "i915_drv.h"
  40 #include "i915_gem_clflush.h"
  41 #include "intel_dsi.h"
  42 #include "i915_trace.h"
  43 #include <drm/drm_atomic.h>
  44 #include <drm/drm_atomic_helper.h>
  45 #include <drm/drm_dp_helper.h>
  46 #include <drm/drm_crtc_helper.h>
  47 #include <drm/drm_plane_helper.h>
  48 #include <drm/drm_rect.h>
  49 #include <drm/drm_atomic_uapi.h>
  50 #include <linux/dma_remapping.h>
  51 #include <linux/reservation.h>
  52
  53 /* Primary plane formats for gen <= 3 */
  54 static const uint32_t i8xx_primary_formats[] = {
  55         DRM_FORMAT_C8,
  56         DRM_FORMAT_RGB565,
  57         DRM_FORMAT_XRGB1555,
  58         DRM_FORMAT_XRGB8888,
  59 };
  60
  61 /* Primary plane formats for gen >= 4 */
  62 static const uint32_t i965_primary_formats[] = {
  63         DRM_FORMAT_C8,
  64         DRM_FORMAT_RGB565,
  65         DRM_FORMAT_XRGB8888,
  66         DRM_FORMAT_XBGR8888,
  67         DRM_FORMAT_XRGB2101010,
  68         DRM_FORMAT_XBGR2101010,
  69 };
  70
  71 static const uint64_t i9xx_format_modifiers[] = {
  72         I915_FORMAT_MOD_X_TILED,
  73         DRM_FORMAT_MOD_LINEAR,
  74         DRM_FORMAT_MOD_INVALID
  75 };
  76
  77 static const uint32_t skl_primary_formats[] = {
  78         DRM_FORMAT_C8,
  79         DRM_FORMAT_RGB565,
  80         DRM_FORMAT_XRGB8888,
  81         DRM_FORMAT_XBGR8888,
  82         DRM_FORMAT_ARGB8888,
  83         DRM_FORMAT_ABGR8888,
  84         DRM_FORMAT_XRGB2101010,
  85         DRM_FORMAT_XBGR2101010,
  86         DRM_FORMAT_YUYV,
  87         DRM_FORMAT_YVYU,
  88         DRM_FORMAT_UYVY,
  89         DRM_FORMAT_VYUY,
  90 };
  91
  92 static const uint32_t skl_pri_planar_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_ARGB8888,
  98         DRM_FORMAT_ABGR8888,
  99         DRM_FORMAT_XRGB2101010,
 100         DRM_FORMAT_XBGR2101010,
 101         DRM_FORMAT_YUYV,
 102         DRM_FORMAT_YVYU,
 103         DRM_FORMAT_UYVY,
 104         DRM_FORMAT_VYUY,
 105         DRM_FORMAT_NV12,
 106 };
 107
 108 static const uint64_t skl_format_modifiers_noccs[] = {
 109         I915_FORMAT_MOD_Yf_TILED,
 110         I915_FORMAT_MOD_Y_TILED,
 111         I915_FORMAT_MOD_X_TILED,
 112         DRM_FORMAT_MOD_LINEAR,
 113         DRM_FORMAT_MOD_INVALID
 114 };
 115
 116 static const uint64_t skl_format_modifiers_ccs[] = {
 117         I915_FORMAT_MOD_Yf_TILED_CCS,
 118         I915_FORMAT_MOD_Y_TILED_CCS,
 119         I915_FORMAT_MOD_Yf_TILED,
 120         I915_FORMAT_MOD_Y_TILED,
 121         I915_FORMAT_MOD_X_TILED,
 122         DRM_FORMAT_MOD_LINEAR,
 123         DRM_FORMAT_MOD_INVALID
 124 };
 125
 126 /* Cursor formats */
 127 static const uint32_t intel_cursor_formats[] = {
 128         DRM_FORMAT_ARGB8888,
 129 };
 130
 131 static const uint64_t cursor_format_modifiers[] = {
 132         DRM_FORMAT_MOD_LINEAR,
 133         DRM_FORMAT_MOD_INVALID
 134 };
 135
 136 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 137                                 struct intel_crtc_state *pipe_config);
 138 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 139                                    struct intel_crtc_state *pipe_config);
 140
 141 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 142                                   struct drm_i915_gem_object *obj,
 143                                   struct drm_mode_fb_cmd2 *mode_cmd);
 144 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
 145 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc);
 146 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 147                                          struct intel_link_m_n *m_n,
 148                                          struct intel_link_m_n *m2_n2);
 149 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 150 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 151 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 152 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state);
 153 static void vlv_prepare_pll(struct intel_crtc *crtc,
 154                             const struct intel_crtc_state *pipe_config);
 155 static void chv_prepare_pll(struct intel_crtc *crtc,
 156                             const struct intel_crtc_state *pipe_config);
 157 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 158 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 159 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 160                                     struct intel_crtc_state *crtc_state);
 161 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 162 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 163 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 164 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 165                                          struct drm_modeset_acquire_ctx *ctx);
 166 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 167
 168 struct intel_limit {
 169         struct {
 170                 int min, max;
 171         } dot, vco, n, m, m1, m2, p, p1;
 172
 173         struct {
 174                 int dot_limit;
 175                 int p2_slow, p2_fast;
 176         } p2;
 177 };
 178
 179 /* returns HPLL frequency in kHz */
 180 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 181 {
 182         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 183
 184         /* Obtain SKU information */
 185         mutex_lock(&dev_priv->sb_lock);
 186         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 187                 CCK_FUSE_HPLL_FREQ_MASK;
 188         mutex_unlock(&dev_priv->sb_lock);
 189
 190         return vco_freq[hpll_freq] * 1000;
 191 }
 192
 193 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 194                       const char *name, u32 reg, int ref_freq)
 195 {
 196         u32 val;
 197         int divider;
 198
 199         mutex_lock(&dev_priv->sb_lock);
 200         val = vlv_cck_read(dev_priv, reg);
 201         mutex_unlock(&dev_priv->sb_lock);
 202
 203         divider = val & CCK_FREQUENCY_VALUES;
 204
 205         WARN((val & CCK_FREQUENCY_STATUS) !=
 206              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 207              "%s change in progress\n", name);
 208
 209         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 210 }
 211
 212 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 213                            const char *name, u32 reg)
 214 {
 215         if (dev_priv->hpll_freq == 0)
 216                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 217
 218         return vlv_get_cck_clock(dev_priv, name, reg,
 219                                  dev_priv->hpll_freq);
 220 }
 221
 222 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 223 {
 224         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 225                 return;
 226
 227         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 228                                                       CCK_CZ_CLOCK_CONTROL);
 229
 230         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 231 }
 232
 233 static inline u32 /* units of 100MHz */
 234 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 235                     const struct intel_crtc_state *pipe_config)
 236 {
 237         if (HAS_DDI(dev_priv))
 238                 return pipe_config->port_clock; /* SPLL */
 239         else
 240                 return dev_priv->fdi_pll_freq;
 241 }
 242
 243 static const struct intel_limit intel_limits_i8xx_dac = {
 244         .dot = { .min = 25000, .max = 350000 },
 245         .vco = { .min = 908000, .max = 1512000 },
 246         .n = { .min = 2, .max = 16 },
 247         .m = { .min = 96, .max = 140 },
 248         .m1 = { .min = 18, .max = 26 },
 249         .m2 = { .min = 6, .max = 16 },
 250         .p = { .min = 4, .max = 128 },
 251         .p1 = { .min = 2, .max = 33 },
 252         .p2 = { .dot_limit = 165000,
 253                 .p2_slow = 4, .p2_fast = 2 },
 254 };
 255
 256 static const struct intel_limit intel_limits_i8xx_dvo = {
 257         .dot = { .min = 25000, .max = 350000 },
 258         .vco = { .min = 908000, .max = 1512000 },
 259         .n = { .min = 2, .max = 16 },
 260         .m = { .min = 96, .max = 140 },
 261         .m1 = { .min = 18, .max = 26 },
 262         .m2 = { .min = 6, .max = 16 },
 263         .p = { .min = 4, .max = 128 },
 264         .p1 = { .min = 2, .max = 33 },
 265         .p2 = { .dot_limit = 165000,
 266                 .p2_slow = 4, .p2_fast = 4 },
 267 };
 268
 269 static const struct intel_limit intel_limits_i8xx_lvds = {
 270         .dot = { .min = 25000, .max = 350000 },
 271         .vco = { .min = 908000, .max = 1512000 },
 272         .n = { .min = 2, .max = 16 },
 273         .m = { .min = 96, .max = 140 },
 274         .m1 = { .min = 18, .max = 26 },
 275         .m2 = { .min = 6, .max = 16 },
 276         .p = { .min = 4, .max = 128 },
 277         .p1 = { .min = 1, .max = 6 },
 278         .p2 = { .dot_limit = 165000,
 279                 .p2_slow = 14, .p2_fast = 7 },
 280 };
 281
 282 static const struct intel_limit intel_limits_i9xx_sdvo = {
 283         .dot = { .min = 20000, .max = 400000 },
 284         .vco = { .min = 1400000, .max = 2800000 },
 285         .n = { .min = 1, .max = 6 },
 286         .m = { .min = 70, .max = 120 },
 287         .m1 = { .min = 8, .max = 18 },
 288         .m2 = { .min = 3, .max = 7 },
 289         .p = { .min = 5, .max = 80 },
 290         .p1 = { .min = 1, .max = 8 },
 291         .p2 = { .dot_limit = 200000,
 292                 .p2_slow = 10, .p2_fast = 5 },
 293 };
 294
 295 static const struct intel_limit intel_limits_i9xx_lvds = {
 296         .dot = { .min = 20000, .max = 400000 },
 297         .vco = { .min = 1400000, .max = 2800000 },
 298         .n = { .min = 1, .max = 6 },
 299         .m = { .min = 70, .max = 120 },
 300         .m1 = { .min = 8, .max = 18 },
 301         .m2 = { .min = 3, .max = 7 },
 302         .p = { .min = 7, .max = 98 },
 303         .p1 = { .min = 1, .max = 8 },
 304         .p2 = { .dot_limit = 112000,
 305                 .p2_slow = 14, .p2_fast = 7 },
 306 };
 307
 308
 309 static const struct intel_limit intel_limits_g4x_sdvo = {
 310         .dot = { .min = 25000, .max = 270000 },
 311         .vco = { .min = 1750000, .max = 3500000},
 312         .n = { .min = 1, .max = 4 },
 313         .m = { .min = 104, .max = 138 },
 314         .m1 = { .min = 17, .max = 23 },
 315         .m2 = { .min = 5, .max = 11 },
 316         .p = { .min = 10, .max = 30 },
 317         .p1 = { .min = 1, .max = 3},
 318         .p2 = { .dot_limit = 270000,
 319                 .p2_slow = 10,
 320                 .p2_fast = 10
 321         },
 322 };
 323
 324 static const struct intel_limit intel_limits_g4x_hdmi = {
 325         .dot = { .min = 22000, .max = 400000 },
 326         .vco = { .min = 1750000, .max = 3500000},
 327         .n = { .min = 1, .max = 4 },
 328         .m = { .min = 104, .max = 138 },
 329         .m1 = { .min = 16, .max = 23 },
 330         .m2 = { .min = 5, .max = 11 },
 331         .p = { .min = 5, .max = 80 },
 332         .p1 = { .min = 1, .max = 8},
 333         .p2 = { .dot_limit = 165000,
 334                 .p2_slow = 10, .p2_fast = 5 },
 335 };
 336
 337 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 338         .dot = { .min = 20000, .max = 115000 },
 339         .vco = { .min = 1750000, .max = 3500000 },
 340         .n = { .min = 1, .max = 3 },
 341         .m = { .min = 104, .max = 138 },
 342         .m1 = { .min = 17, .max = 23 },
 343         .m2 = { .min = 5, .max = 11 },
 344         .p = { .min = 28, .max = 112 },
 345         .p1 = { .min = 2, .max = 8 },
 346         .p2 = { .dot_limit = 0,
 347                 .p2_slow = 14, .p2_fast = 14
 348         },
 349 };
 350
 351 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 352         .dot = { .min = 80000, .max = 224000 },
 353         .vco = { .min = 1750000, .max = 3500000 },
 354         .n = { .min = 1, .max = 3 },
 355         .m = { .min = 104, .max = 138 },
 356         .m1 = { .min = 17, .max = 23 },
 357         .m2 = { .min = 5, .max = 11 },
 358         .p = { .min = 14, .max = 42 },
 359         .p1 = { .min = 2, .max = 6 },
 360         .p2 = { .dot_limit = 0,
 361                 .p2_slow = 7, .p2_fast = 7
 362         },
 363 };
 364
 365 static const struct intel_limit intel_limits_pineview_sdvo = {
 366         .dot = { .min = 20000, .max = 400000},
 367         .vco = { .min = 1700000, .max = 3500000 },
 368         /* Pineview's Ncounter is a ring counter */
 369         .n = { .min = 3, .max = 6 },
 370         .m = { .min = 2, .max = 256 },
 371         /* Pineview only has one combined m divider, which we treat as m2. */
 372         .m1 = { .min = 0, .max = 0 },
 373         .m2 = { .min = 0, .max = 254 },
 374         .p = { .min = 5, .max = 80 },
 375         .p1 = { .min = 1, .max = 8 },
 376         .p2 = { .dot_limit = 200000,
 377                 .p2_slow = 10, .p2_fast = 5 },
 378 };
 379
 380 static const struct intel_limit intel_limits_pineview_lvds = {
 381         .dot = { .min = 20000, .max = 400000 },
 382         .vco = { .min = 1700000, .max = 3500000 },
 383         .n = { .min = 3, .max = 6 },
 384         .m = { .min = 2, .max = 256 },
 385         .m1 = { .min = 0, .max = 0 },
 386         .m2 = { .min = 0, .max = 254 },
 387         .p = { .min = 7, .max = 112 },
 388         .p1 = { .min = 1, .max = 8 },
 389         .p2 = { .dot_limit = 112000,
 390                 .p2_slow = 14, .p2_fast = 14 },
 391 };
 392
 393 /* Ironlake / Sandybridge
 394  *
 395  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 396  * the range value for them is (actual_value - 2).
 397  */
 398 static const struct intel_limit intel_limits_ironlake_dac = {
 399         .dot = { .min = 25000, .max = 350000 },
 400         .vco = { .min = 1760000, .max = 3510000 },
 401         .n = { .min = 1, .max = 5 },
 402         .m = { .min = 79, .max = 127 },
 403         .m1 = { .min = 12, .max = 22 },
 404         .m2 = { .min = 5, .max = 9 },
 405         .p = { .min = 5, .max = 80 },
 406         .p1 = { .min = 1, .max = 8 },
 407         .p2 = { .dot_limit = 225000,
 408                 .p2_slow = 10, .p2_fast = 5 },
 409 };
 410
 411 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 412         .dot = { .min = 25000, .max = 350000 },
 413         .vco = { .min = 1760000, .max = 3510000 },
 414         .n = { .min = 1, .max = 3 },
 415         .m = { .min = 79, .max = 118 },
 416         .m1 = { .min = 12, .max = 22 },
 417         .m2 = { .min = 5, .max = 9 },
 418         .p = { .min = 28, .max = 112 },
 419         .p1 = { .min = 2, .max = 8 },
 420         .p2 = { .dot_limit = 225000,
 421                 .p2_slow = 14, .p2_fast = 14 },
 422 };
 423
 424 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 425         .dot = { .min = 25000, .max = 350000 },
 426         .vco = { .min = 1760000, .max = 3510000 },
 427         .n = { .min = 1, .max = 3 },
 428         .m = { .min = 79, .max = 127 },
 429         .m1 = { .min = 12, .max = 22 },
 430         .m2 = { .min = 5, .max = 9 },
 431         .p = { .min = 14, .max = 56 },
 432         .p1 = { .min = 2, .max = 8 },
 433         .p2 = { .dot_limit = 225000,
 434                 .p2_slow = 7, .p2_fast = 7 },
 435 };
 436
 437 /* LVDS 100mhz refclk limits. */
 438 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 439         .dot = { .min = 25000, .max = 350000 },
 440         .vco = { .min = 1760000, .max = 3510000 },
 441         .n = { .min = 1, .max = 2 },
 442         .m = { .min = 79, .max = 126 },
 443         .m1 = { .min = 12, .max = 22 },
 444         .m2 = { .min = 5, .max = 9 },
 445         .p = { .min = 28, .max = 112 },
 446         .p1 = { .min = 2, .max = 8 },
 447         .p2 = { .dot_limit = 225000,
 448                 .p2_slow = 14, .p2_fast = 14 },
 449 };
 450
 451 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 452         .dot = { .min = 25000, .max = 350000 },
 453         .vco = { .min = 1760000, .max = 3510000 },
 454         .n = { .min = 1, .max = 3 },
 455         .m = { .min = 79, .max = 126 },
 456         .m1 = { .min = 12, .max = 22 },
 457         .m2 = { .min = 5, .max = 9 },
 458         .p = { .min = 14, .max = 42 },
 459         .p1 = { .min = 2, .max = 6 },
 460         .p2 = { .dot_limit = 225000,
 461                 .p2_slow = 7, .p2_fast = 7 },
 462 };
 463
 464 static const struct intel_limit intel_limits_vlv = {
 465          /*
 466           * These are the data rate limits (measured in fast clocks)
 467           * since those are the strictest limits we have. The fast
 468           * clock and actual rate limits are more relaxed, so checking
 469           * them would make no difference.
 470           */
 471         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 472         .vco = { .min = 4000000, .max = 6000000 },
 473         .n = { .min = 1, .max = 7 },
 474         .m1 = { .min = 2, .max = 3 },
 475         .m2 = { .min = 11, .max = 156 },
 476         .p1 = { .min = 2, .max = 3 },
 477         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 478 };
 479
 480 static const struct intel_limit intel_limits_chv = {
 481         /*
 482          * These are the data rate limits (measured in fast clocks)
 483          * since those are the strictest limits we have.  The fast
 484          * clock and actual rate limits are more relaxed, so checking
 485          * them would make no difference.
 486          */
 487         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 488         .vco = { .min = 4800000, .max = 6480000 },
 489         .n = { .min = 1, .max = 1 },
 490         .m1 = { .min = 2, .max = 2 },
 491         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 492         .p1 = { .min = 2, .max = 4 },
 493         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 494 };
 495
 496 static const struct intel_limit intel_limits_bxt = {
 497         /* FIXME: find real dot limits */
 498         .dot = { .min = 0, .max = INT_MAX },
 499         .vco = { .min = 4800000, .max = 6700000 },
 500         .n = { .min = 1, .max = 1 },
 501         .m1 = { .min = 2, .max = 2 },
 502         /* FIXME: find real m2 limits */
 503         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 504         .p1 = { .min = 2, .max = 4 },
 505         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 506 };
 507
 508 static void
 509 skl_wa_528(struct drm_i915_private *dev_priv, int pipe, bool enable)
 510 {
 511         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
 512                 return;
 513
 514         if (enable)
 515                 I915_WRITE(CHICKEN_PIPESL_1(pipe), HSW_FBCQ_DIS);
 516         else
 517                 I915_WRITE(CHICKEN_PIPESL_1(pipe), 0);
 518 }
 519
 520 static void
 521 skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
 522 {
 523         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
 524                 return;
 525
 526         if (enable)
 527                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 528                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 529         else
 530                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 531                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 532                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 533 }
 534
 535 static bool
 536 needs_modeset(const struct drm_crtc_state *state)
 537 {
 538         return drm_atomic_crtc_needs_modeset(state);
 539 }
 540
 541 /*
 542  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 543  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 544  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 545  * The helpers' return value is the rate of the clock that is fed to the
 546  * display engine's pipe which can be the above fast dot clock rate or a
 547  * divided-down version of it.
 548  */
 549 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 550 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = clock->m2 + 2;
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 563 {
 564         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 565 }
 566
 567 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 568 {
 569         clock->m = i9xx_dpll_compute_m(clock);
 570         clock->p = clock->p1 * clock->p2;
 571         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 572                 return 0;
 573         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 574         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 575
 576         return clock->dot;
 577 }
 578
 579 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 580 {
 581         clock->m = clock->m1 * clock->m2;
 582         clock->p = clock->p1 * clock->p2;
 583         if (WARN_ON(clock->n == 0 || clock->p == 0))
 584                 return 0;
 585         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 586         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 587
 588         return clock->dot / 5;
 589 }
 590
 591 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 592 {
 593         clock->m = clock->m1 * clock->m2;
 594         clock->p = clock->p1 * clock->p2;
 595         if (WARN_ON(clock->n == 0 || clock->p == 0))
 596                 return 0;
 597         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 598                         clock->n << 22);
 599         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 600
 601         return clock->dot / 5;
 602 }
 603
 604 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 605
 606 /*
 607  * Returns whether the given set of divisors are valid for a given refclk with
 608  * the given connectors.
 609  */
 610 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 611                                const struct intel_limit *limit,
 612                                const struct dpll *clock)
 613 {
 614         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 615                 INTELPllInvalid("n out of range\n");
 616         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 617                 INTELPllInvalid("p1 out of range\n");
 618         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 619                 INTELPllInvalid("m2 out of range\n");
 620         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 621                 INTELPllInvalid("m1 out of range\n");
 622
 623         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 624             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 625                 if (clock->m1 <= clock->m2)
 626                         INTELPllInvalid("m1 <= m2\n");
 627
 628         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 629             !IS_GEN9_LP(dev_priv)) {
 630                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 631                         INTELPllInvalid("p out of range\n");
 632                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 633                         INTELPllInvalid("m out of range\n");
 634         }
 635
 636         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 637                 INTELPllInvalid("vco out of range\n");
 638         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 639          * connector, etc., rather than just a single range.
 640          */
 641         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 642                 INTELPllInvalid("dot out of range\n");
 643
 644         return true;
 645 }
 646
 647 static int
 648 i9xx_select_p2_div(const struct intel_limit *limit,
 649                    const struct intel_crtc_state *crtc_state,
 650                    int target)
 651 {
 652         struct drm_device *dev = crtc_state->base.crtc->dev;
 653
 654         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 655                 /*
 656                  * For LVDS just rely on its current settings for dual-channel.
 657                  * We haven't figured out how to reliably set up different
 658                  * single/dual channel state, if we even can.
 659                  */
 660                 if (intel_is_dual_link_lvds(dev))
 661                         return limit->p2.p2_fast;
 662                 else
 663                         return limit->p2.p2_slow;
 664         } else {
 665                 if (target < limit->p2.dot_limit)
 666                         return limit->p2.p2_slow;
 667                 else
 668                         return limit->p2.p2_fast;
 669         }
 670 }
 671
 672 /*
 673  * Returns a set of divisors for the desired target clock with the given
 674  * refclk, or FALSE.  The returned values represent the clock equation:
 675  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 676  *
 677  * Target and reference clocks are specified in kHz.
 678  *
 679  * If match_clock is provided, then best_clock P divider must match the P
 680  * divider from @match_clock used for LVDS downclocking.
 681  */
 682 static bool
 683 i9xx_find_best_dpll(const struct intel_limit *limit,
 684                     struct intel_crtc_state *crtc_state,
 685                     int target, int refclk, struct dpll *match_clock,
 686                     struct dpll *best_clock)
 687 {
 688         struct drm_device *dev = crtc_state->base.crtc->dev;
 689         struct dpll clock;
 690         int err = target;
 691
 692         memset(best_clock, 0, sizeof(*best_clock));
 693
 694         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 695
 696         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 697              clock.m1++) {
 698                 for (clock.m2 = limit->m2.min;
 699                      clock.m2 <= limit->m2.max; clock.m2++) {
 700                         if (clock.m2 >= clock.m1)
 701                                 break;
 702                         for (clock.n = limit->n.min;
 703                              clock.n <= limit->n.max; clock.n++) {
 704                                 for (clock.p1 = limit->p1.min;
 705                                         clock.p1 <= limit->p1.max; clock.p1++) {
 706                                         int this_err;
 707
 708                                         i9xx_calc_dpll_params(refclk, &clock);
 709                                         if (!intel_PLL_is_valid(to_i915(dev),
 710                                                                 limit,
 711                                                                 &clock))
 712                                                 continue;
 713                                         if (match_clock &&
 714                                             clock.p != match_clock->p)
 715                                                 continue;
 716
 717                                         this_err = abs(clock.dot - target);
 718                                         if (this_err < err) {
 719                                                 *best_clock = clock;
 720                                                 err = this_err;
 721                                         }
 722                                 }
 723                         }
 724                 }
 725         }
 726
 727         return (err != target);
 728 }
 729
 730 /*
 731  * Returns a set of divisors for the desired target clock with the given
 732  * refclk, or FALSE.  The returned values represent the clock equation:
 733  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 734  *
 735  * Target and reference clocks are specified in kHz.
 736  *
 737  * If match_clock is provided, then best_clock P divider must match the P
 738  * divider from @match_clock used for LVDS downclocking.
 739  */
 740 static bool
 741 pnv_find_best_dpll(const struct intel_limit *limit,
 742                    struct intel_crtc_state *crtc_state,
 743                    int target, int refclk, struct dpll *match_clock,
 744                    struct dpll *best_clock)
 745 {
 746         struct drm_device *dev = crtc_state->base.crtc->dev;
 747         struct dpll clock;
 748         int err = target;
 749
 750         memset(best_clock, 0, sizeof(*best_clock));
 751
 752         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 753
 754         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 755              clock.m1++) {
 756                 for (clock.m2 = limit->m2.min;
 757                      clock.m2 <= limit->m2.max; clock.m2++) {
 758                         for (clock.n = limit->n.min;
 759                              clock.n <= limit->n.max; clock.n++) {
 760                                 for (clock.p1 = limit->p1.min;
 761                                         clock.p1 <= limit->p1.max; clock.p1++) {
 762                                         int this_err;
 763
 764                                         pnv_calc_dpll_params(refclk, &clock);
 765                                         if (!intel_PLL_is_valid(to_i915(dev),
 766                                                                 limit,
 767                                                                 &clock))
 768                                                 continue;
 769                                         if (match_clock &&
 770                                             clock.p != match_clock->p)
 771                                                 continue;
 772
 773                                         this_err = abs(clock.dot - target);
 774                                         if (this_err < err) {
 775                                                 *best_clock = clock;
 776                                                 err = this_err;
 777                                         }
 778                                 }
 779                         }
 780                 }
 781         }
 782
 783         return (err != target);
 784 }
 785
 786 /*
 787  * Returns a set of divisors for the desired target clock with the given
 788  * refclk, or FALSE.  The returned values represent the clock equation:
 789  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 790  *
 791  * Target and reference clocks are specified in kHz.
 792  *
 793  * If match_clock is provided, then best_clock P divider must match the P
 794  * divider from @match_clock used for LVDS downclocking.
 795  */
 796 static bool
 797 g4x_find_best_dpll(const struct intel_limit *limit,
 798                    struct intel_crtc_state *crtc_state,
 799                    int target, int refclk, struct dpll *match_clock,
 800                    struct dpll *best_clock)
 801 {
 802         struct drm_device *dev = crtc_state->base.crtc->dev;
 803         struct dpll clock;
 804         int max_n;
 805         bool found = false;
 806         /* approximately equals target * 0.00585 */
 807         int err_most = (target >> 8) + (target >> 9);
 808
 809         memset(best_clock, 0, sizeof(*best_clock));
 810
 811         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 812
 813         max_n = limit->n.max;
 814         /* based on hardware requirement, prefer smaller n to precision */
 815         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 816                 /* based on hardware requirement, prefere larger m1,m2 */
 817                 for (clock.m1 = limit->m1.max;
 818                      clock.m1 >= limit->m1.min; clock.m1--) {
 819                         for (clock.m2 = limit->m2.max;
 820                              clock.m2 >= limit->m2.min; clock.m2--) {
 821                                 for (clock.p1 = limit->p1.max;
 822                                      clock.p1 >= limit->p1.min; clock.p1--) {
 823                                         int this_err;
 824
 825                                         i9xx_calc_dpll_params(refclk, &clock);
 826                                         if (!intel_PLL_is_valid(to_i915(dev),
 827                                                                 limit,
 828                                                                 &clock))
 829                                                 continue;
 830
 831                                         this_err = abs(clock.dot - target);
 832                                         if (this_err < err_most) {
 833                                                 *best_clock = clock;
 834                                                 err_most = this_err;
 835                                                 max_n = clock.n;
 836                                                 found = true;
 837                                         }
 838                                 }
 839                         }
 840                 }
 841         }
 842         return found;
 843 }
 844
 845 /*
 846  * Check if the calculated PLL configuration is more optimal compared to the
 847  * best configuration and error found so far. Return the calculated error.
 848  */
 849 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 850                                const struct dpll *calculated_clock,
 851                                const struct dpll *best_clock,
 852                                unsigned int best_error_ppm,
 853                                unsigned int *error_ppm)
 854 {
 855         /*
 856          * For CHV ignore the error and consider only the P value.
 857          * Prefer a bigger P value based on HW requirements.
 858          */
 859         if (IS_CHERRYVIEW(to_i915(dev))) {
 860                 *error_ppm = 0;
 861
 862                 return calculated_clock->p > best_clock->p;
 863         }
 864
 865         if (WARN_ON_ONCE(!target_freq))
 866                 return false;
 867
 868         *error_ppm = div_u64(1000000ULL *
 869                                 abs(target_freq - calculated_clock->dot),
 870                              target_freq);
 871         /*
 872          * Prefer a better P value over a better (smaller) error if the error
 873          * is small. Ensure this preference for future configurations too by
 874          * setting the error to 0.
 875          */
 876         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 877                 *error_ppm = 0;
 878
 879                 return true;
 880         }
 881
 882         return *error_ppm + 10 < best_error_ppm;
 883 }
 884
 885 /*
 886  * Returns a set of divisors for the desired target clock with the given
 887  * refclk, or FALSE.  The returned values represent the clock equation:
 888  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 889  */
 890 static bool
 891 vlv_find_best_dpll(const struct intel_limit *limit,
 892                    struct intel_crtc_state *crtc_state,
 893                    int target, int refclk, struct dpll *match_clock,
 894                    struct dpll *best_clock)
 895 {
 896         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 897         struct drm_device *dev = crtc->base.dev;
 898         struct dpll clock;
 899         unsigned int bestppm = 1000000;
 900         /* min update 19.2 MHz */
 901         int max_n = min(limit->n.max, refclk / 19200);
 902         bool found = false;
 903
 904         target *= 5; /* fast clock */
 905
 906         memset(best_clock, 0, sizeof(*best_clock));
 907
 908         /* based on hardware requirement, prefer smaller n to precision */
 909         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 910                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 911                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 912                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 913                                 clock.p = clock.p1 * clock.p2;
 914                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 915                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 916                                         unsigned int ppm;
 917
 918                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 919                                                                      refclk * clock.m1);
 920
 921                                         vlv_calc_dpll_params(refclk, &clock);
 922
 923                                         if (!intel_PLL_is_valid(to_i915(dev),
 924                                                                 limit,
 925                                                                 &clock))
 926                                                 continue;
 927
 928                                         if (!vlv_PLL_is_optimal(dev, target,
 929                                                                 &clock,
 930                                                                 best_clock,
 931                                                                 bestppm, &ppm))
 932                                                 continue;
 933
 934                                         *best_clock = clock;
 935                                         bestppm = ppm;
 936                                         found = true;
 937                                 }
 938                         }
 939                 }
 940         }
 941
 942         return found;
 943 }
 944
 945 /*
 946  * Returns a set of divisors for the desired target clock with the given
 947  * refclk, or FALSE.  The returned values represent the clock equation:
 948  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 949  */
 950 static bool
 951 chv_find_best_dpll(const struct intel_limit *limit,
 952                    struct intel_crtc_state *crtc_state,
 953                    int target, int refclk, struct dpll *match_clock,
 954                    struct dpll *best_clock)
 955 {
 956         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 957         struct drm_device *dev = crtc->base.dev;
 958         unsigned int best_error_ppm;
 959         struct dpll clock;
 960         uint64_t m2;
 961         int found = false;
 962
 963         memset(best_clock, 0, sizeof(*best_clock));
 964         best_error_ppm = 1000000;
 965
 966         /*
 967          * Based on hardware doc, the n always set to 1, and m1 always
 968          * set to 2.  If requires to support 200Mhz refclk, we need to
 969          * revisit this because n may not 1 anymore.
 970          */
 971         clock.n = 1, clock.m1 = 2;
 972         target *= 5;    /* fast clock */
 973
 974         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 975                 for (clock.p2 = limit->p2.p2_fast;
 976                                 clock.p2 >= limit->p2.p2_slow;
 977                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 978                         unsigned int error_ppm;
 979
 980                         clock.p = clock.p1 * clock.p2;
 981
 982                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
 983                                         clock.n) << 22, refclk * clock.m1);
 984
 985                         if (m2 > INT_MAX/clock.m1)
 986                                 continue;
 987
 988                         clock.m2 = m2;
 989
 990                         chv_calc_dpll_params(refclk, &clock);
 991
 992                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 993                                 continue;
 994
 995                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 996                                                 best_error_ppm, &error_ppm))
 997                                 continue;
 998
 999                         *best_clock = clock;
1000                         best_error_ppm = error_ppm;
1001                         found = true;
1002                 }
1003         }
1004
1005         return found;
1006 }
1007
1008 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
1009                         struct dpll *best_clock)
1010 {
1011         int refclk = 100000;
1012         const struct intel_limit *limit = &intel_limits_bxt;
1013
1014         return chv_find_best_dpll(limit, crtc_state,
1015                                   target_clock, refclk, NULL, best_clock);
1016 }
1017
1018 bool intel_crtc_active(struct intel_crtc *crtc)
1019 {
1020         /* Be paranoid as we can arrive here with only partial
1021          * state retrieved from the hardware during setup.
1022          *
1023          * We can ditch the adjusted_mode.crtc_clock check as soon
1024          * as Haswell has gained clock readout/fastboot support.
1025          *
1026          * We can ditch the crtc->primary->state->fb check as soon as we can
1027          * properly reconstruct framebuffers.
1028          *
1029          * FIXME: The intel_crtc->active here should be switched to
1030          * crtc->state->active once we have proper CRTC states wired up
1031          * for atomic.
1032          */
1033         return crtc->active && crtc->base.primary->state->fb &&
1034                 crtc->config->base.adjusted_mode.crtc_clock;
1035 }
1036
1037 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1038                                              enum pipe pipe)
1039 {
1040         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1041
1042         return crtc->config->cpu_transcoder;
1043 }
1044
1045 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1046                                     enum pipe pipe)
1047 {
1048         i915_reg_t reg = PIPEDSL(pipe);
1049         u32 line1, line2;
1050         u32 line_mask;
1051
1052         if (IS_GEN2(dev_priv))
1053                 line_mask = DSL_LINEMASK_GEN2;
1054         else
1055                 line_mask = DSL_LINEMASK_GEN3;
1056
1057         line1 = I915_READ(reg) & line_mask;
1058         msleep(5);
1059         line2 = I915_READ(reg) & line_mask;
1060
1061         return line1 != line2;
1062 }
1063
1064 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1065 {
1066         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1067         enum pipe pipe = crtc->pipe;
1068
1069         /* Wait for the display line to settle/start moving */
1070         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1071                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1072                           pipe_name(pipe), onoff(state));
1073 }
1074
1075 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1076 {
1077         wait_for_pipe_scanline_moving(crtc, false);
1078 }
1079
1080 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1081 {
1082         wait_for_pipe_scanline_moving(crtc, true);
1083 }
1084
1085 static void
1086 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1087 {
1088         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1089         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1090
1091         if (INTEL_GEN(dev_priv) >= 4) {
1092                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1093                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1094
1095                 /* Wait for the Pipe State to go off */
1096                 if (intel_wait_for_register(dev_priv,
1097                                             reg, I965_PIPECONF_ACTIVE, 0,
1098                                             100))
1099                         WARN(1, "pipe_off wait timed out\n");
1100         } else {
1101                 intel_wait_for_pipe_scanline_stopped(crtc);
1102         }
1103 }
1104
1105 /* Only for pre-ILK configs */
1106 void assert_pll(struct drm_i915_private *dev_priv,
1107                 enum pipe pipe, bool state)
1108 {
1109         u32 val;
1110         bool cur_state;
1111
1112         val = I915_READ(DPLL(pipe));
1113         cur_state = !!(val & DPLL_VCO_ENABLE);
1114         I915_STATE_WARN(cur_state != state,
1115              "PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 /* XXX: the dsi pll is shared between MIPI DSI ports */
1120 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1121 {
1122         u32 val;
1123         bool cur_state;
1124
1125         mutex_lock(&dev_priv->sb_lock);
1126         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1127         mutex_unlock(&dev_priv->sb_lock);
1128
1129         cur_state = val & DSI_PLL_VCO_EN;
1130         I915_STATE_WARN(cur_state != state,
1131              "DSI PLL state assertion failure (expected %s, current %s)\n",
1132                         onoff(state), onoff(cur_state));
1133 }
1134
1135 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1136                           enum pipe pipe, bool state)
1137 {
1138         bool cur_state;
1139         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1140                                                                       pipe);
1141
1142         if (HAS_DDI(dev_priv)) {
1143                 /* DDI does not have a specific FDI_TX register */
1144                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1145                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1146         } else {
1147                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1148                 cur_state = !!(val & FDI_TX_ENABLE);
1149         }
1150         I915_STATE_WARN(cur_state != state,
1151              "FDI TX state assertion failure (expected %s, current %s)\n",
1152                         onoff(state), onoff(cur_state));
1153 }
1154 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1155 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1156
1157 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1158                           enum pipe pipe, bool state)
1159 {
1160         u32 val;
1161         bool cur_state;
1162
1163         val = I915_READ(FDI_RX_CTL(pipe));
1164         cur_state = !!(val & FDI_RX_ENABLE);
1165         I915_STATE_WARN(cur_state != state,
1166              "FDI RX state assertion failure (expected %s, current %s)\n",
1167                         onoff(state), onoff(cur_state));
1168 }
1169 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1170 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1171
1172 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1173                                       enum pipe pipe)
1174 {
1175         u32 val;
1176
1177         /* ILK FDI PLL is always enabled */
1178         if (IS_GEN5(dev_priv))
1179                 return;
1180
1181         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1182         if (HAS_DDI(dev_priv))
1183                 return;
1184
1185         val = I915_READ(FDI_TX_CTL(pipe));
1186         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1187 }
1188
1189 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1190                        enum pipe pipe, bool state)
1191 {
1192         u32 val;
1193         bool cur_state;
1194
1195         val = I915_READ(FDI_RX_CTL(pipe));
1196         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1197         I915_STATE_WARN(cur_state != state,
1198              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1199                         onoff(state), onoff(cur_state));
1200 }
1201
1202 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1203 {
1204         i915_reg_t pp_reg;
1205         u32 val;
1206         enum pipe panel_pipe = INVALID_PIPE;
1207         bool locked = true;
1208
1209         if (WARN_ON(HAS_DDI(dev_priv)))
1210                 return;
1211
1212         if (HAS_PCH_SPLIT(dev_priv)) {
1213                 u32 port_sel;
1214
1215                 pp_reg = PP_CONTROL(0);
1216                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1217
1218                 switch (port_sel) {
1219                 case PANEL_PORT_SELECT_LVDS:
1220                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1221                         break;
1222                 case PANEL_PORT_SELECT_DPA:
1223                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1224                         break;
1225                 case PANEL_PORT_SELECT_DPC:
1226                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1227                         break;
1228                 case PANEL_PORT_SELECT_DPD:
1229                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1230                         break;
1231                 default:
1232                         MISSING_CASE(port_sel);
1233                         break;
1234                 }
1235         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1236                 /* presumably write lock depends on pipe, not port select */
1237                 pp_reg = PP_CONTROL(pipe);
1238                 panel_pipe = pipe;
1239         } else {
1240                 u32 port_sel;
1241
1242                 pp_reg = PP_CONTROL(0);
1243                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1244
1245                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1246                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1247         }
1248
1249         val = I915_READ(pp_reg);
1250         if (!(val & PANEL_POWER_ON) ||
1251             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1252                 locked = false;
1253
1254         I915_STATE_WARN(panel_pipe == pipe && locked,
1255              "panel assertion failure, pipe %c regs locked\n",
1256              pipe_name(pipe));
1257 }
1258
1259 void assert_pipe(struct drm_i915_private *dev_priv,
1260                  enum pipe pipe, bool state)
1261 {
1262         bool cur_state;
1263         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1264                                                                       pipe);
1265         enum intel_display_power_domain power_domain;
1266
1267         /* we keep both pipes enabled on 830 */
1268         if (IS_I830(dev_priv))
1269                 state = true;
1270
1271         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1272         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
1273                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1274                 cur_state = !!(val & PIPECONF_ENABLE);
1275
1276                 intel_display_power_put(dev_priv, power_domain);
1277         } else {
1278                 cur_state = false;
1279         }
1280
1281         I915_STATE_WARN(cur_state != state,
1282              "pipe %c assertion failure (expected %s, current %s)\n",
1283                         pipe_name(pipe), onoff(state), onoff(cur_state));
1284 }
1285
1286 static void assert_plane(struct intel_plane *plane, bool state)
1287 {
1288         enum pipe pipe;
1289         bool cur_state;
1290
1291         cur_state = plane->get_hw_state(plane, &pipe);
1292
1293         I915_STATE_WARN(cur_state != state,
1294                         "%s assertion failure (expected %s, current %s)\n",
1295                         plane->base.name, onoff(state), onoff(cur_state));
1296 }
1297
1298 #define assert_plane_enabled(p) assert_plane(p, true)
1299 #define assert_plane_disabled(p) assert_plane(p, false)
1300
1301 static void assert_planes_disabled(struct intel_crtc *crtc)
1302 {
1303         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1304         struct intel_plane *plane;
1305
1306         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1307                 assert_plane_disabled(plane);
1308 }
1309
1310 static void assert_vblank_disabled(struct drm_crtc *crtc)
1311 {
1312         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1313                 drm_crtc_vblank_put(crtc);
1314 }
1315
1316 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1317                                     enum pipe pipe)
1318 {
1319         u32 val;
1320         bool enabled;
1321
1322         val = I915_READ(PCH_TRANSCONF(pipe));
1323         enabled = !!(val & TRANS_ENABLE);
1324         I915_STATE_WARN(enabled,
1325              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1326              pipe_name(pipe));
1327 }
1328
1329 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1330                                    enum pipe pipe, enum port port,
1331                                    i915_reg_t dp_reg)
1332 {
1333         enum pipe port_pipe;
1334         bool state;
1335
1336         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1337
1338         I915_STATE_WARN(state && port_pipe == pipe,
1339                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1340                         port_name(port), pipe_name(pipe));
1341
1342         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1343                         "IBX PCH DP %c still using transcoder B\n",
1344                         port_name(port));
1345 }
1346
1347 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1348                                      enum pipe pipe, enum port port,
1349                                      i915_reg_t hdmi_reg)
1350 {
1351         enum pipe port_pipe;
1352         bool state;
1353
1354         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1355
1356         I915_STATE_WARN(state && port_pipe == pipe,
1357                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1358                         port_name(port), pipe_name(pipe));
1359
1360         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1361                         "IBX PCH HDMI %c still using transcoder B\n",
1362                         port_name(port));
1363 }
1364
1365 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1366                                       enum pipe pipe)
1367 {
1368         enum pipe port_pipe;
1369
1370         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1371         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1372         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1373
1374         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1375                         port_pipe == pipe,
1376                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1377                         pipe_name(pipe));
1378
1379         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1380                         port_pipe == pipe,
1381                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1382                         pipe_name(pipe));
1383
1384         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1385         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1386         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1387 }
1388
1389 static void _vlv_enable_pll(struct intel_crtc *crtc,
1390                             const struct intel_crtc_state *pipe_config)
1391 {
1392         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1393         enum pipe pipe = crtc->pipe;
1394
1395         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1396         POSTING_READ(DPLL(pipe));
1397         udelay(150);
1398
1399         if (intel_wait_for_register(dev_priv,
1400                                     DPLL(pipe),
1401                                     DPLL_LOCK_VLV,
1402                                     DPLL_LOCK_VLV,
1403                                     1))
1404                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1405 }
1406
1407 static void vlv_enable_pll(struct intel_crtc *crtc,
1408                            const struct intel_crtc_state *pipe_config)
1409 {
1410         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1411         enum pipe pipe = crtc->pipe;
1412
1413         assert_pipe_disabled(dev_priv, pipe);
1414
1415         /* PLL is protected by panel, make sure we can write it */
1416         assert_panel_unlocked(dev_priv, pipe);
1417
1418         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1419                 _vlv_enable_pll(crtc, pipe_config);
1420
1421         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1422         POSTING_READ(DPLL_MD(pipe));
1423 }
1424
1425
1426 static void _chv_enable_pll(struct intel_crtc *crtc,
1427                             const struct intel_crtc_state *pipe_config)
1428 {
1429         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1430         enum pipe pipe = crtc->pipe;
1431         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1432         u32 tmp;
1433
1434         mutex_lock(&dev_priv->sb_lock);
1435
1436         /* Enable back the 10bit clock to display controller */
1437         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1438         tmp |= DPIO_DCLKP_EN;
1439         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1440
1441         mutex_unlock(&dev_priv->sb_lock);
1442
1443         /*
1444          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1445          */
1446         udelay(1);
1447
1448         /* Enable PLL */
1449         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1450
1451         /* Check PLL is locked */
1452         if (intel_wait_for_register(dev_priv,
1453                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1454                                     1))
1455                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1456 }
1457
1458 static void chv_enable_pll(struct intel_crtc *crtc,
1459                            const struct intel_crtc_state *pipe_config)
1460 {
1461         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1462         enum pipe pipe = crtc->pipe;
1463
1464         assert_pipe_disabled(dev_priv, pipe);
1465
1466         /* PLL is protected by panel, make sure we can write it */
1467         assert_panel_unlocked(dev_priv, pipe);
1468
1469         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1470                 _chv_enable_pll(crtc, pipe_config);
1471
1472         if (pipe != PIPE_A) {
1473                 /*
1474                  * WaPixelRepeatModeFixForC0:chv
1475                  *
1476                  * DPLLCMD is AWOL. Use chicken bits to propagate
1477                  * the value from DPLLBMD to either pipe B or C.
1478                  */
1479                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1480                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1481                 I915_WRITE(CBR4_VLV, 0);
1482                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1483
1484                 /*
1485                  * DPLLB VGA mode also seems to cause problems.
1486                  * We should always have it disabled.
1487                  */
1488                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1489         } else {
1490                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1491                 POSTING_READ(DPLL_MD(pipe));
1492         }
1493 }
1494
1495 static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
1496 {
1497         struct intel_crtc *crtc;
1498         int count = 0;
1499
1500         for_each_intel_crtc(&dev_priv->drm, crtc) {
1501                 count += crtc->base.state->active &&
1502                         intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
1503         }
1504
1505         return count;
1506 }
1507
1508 static void i9xx_enable_pll(struct intel_crtc *crtc,
1509                             const struct intel_crtc_state *crtc_state)
1510 {
1511         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1512         i915_reg_t reg = DPLL(crtc->pipe);
1513         u32 dpll = crtc_state->dpll_hw_state.dpll;
1514         int i;
1515
1516         assert_pipe_disabled(dev_priv, crtc->pipe);
1517
1518         /* PLL is protected by panel, make sure we can write it */
1519         if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
1520                 assert_panel_unlocked(dev_priv, crtc->pipe);
1521
1522         /* Enable DVO 2x clock on both PLLs if necessary */
1523         if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
1524                 /*
1525                  * It appears to be important that we don't enable this
1526                  * for the current pipe before otherwise configuring the
1527                  * PLL. No idea how this should be handled if multiple
1528                  * DVO outputs are enabled simultaneosly.
1529                  */
1530                 dpll |= DPLL_DVO_2X_MODE;
1531                 I915_WRITE(DPLL(!crtc->pipe),
1532                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1533         }
1534
1535         /*
1536          * Apparently we need to have VGA mode enabled prior to changing
1537          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1538          * dividers, even though the register value does change.
1539          */
1540         I915_WRITE(reg, 0);
1541
1542         I915_WRITE(reg, dpll);
1543
1544         /* Wait for the clocks to stabilize. */
1545         POSTING_READ(reg);
1546         udelay(150);
1547
1548         if (INTEL_GEN(dev_priv) >= 4) {
1549                 I915_WRITE(DPLL_MD(crtc->pipe),
1550                            crtc_state->dpll_hw_state.dpll_md);
1551         } else {
1552                 /* The pixel multiplier can only be updated once the
1553                  * DPLL is enabled and the clocks are stable.
1554                  *
1555                  * So write it again.
1556                  */
1557                 I915_WRITE(reg, dpll);
1558         }
1559
1560         /* We do this three times for luck */
1561         for (i = 0; i < 3; i++) {
1562                 I915_WRITE(reg, dpll);
1563                 POSTING_READ(reg);
1564                 udelay(150); /* wait for warmup */
1565         }
1566 }
1567
1568 static void i9xx_disable_pll(struct intel_crtc *crtc)
1569 {
1570         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1571         enum pipe pipe = crtc->pipe;
1572
1573         /* Disable DVO 2x clock on both PLLs if necessary */
1574         if (IS_I830(dev_priv) &&
1575             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
1576             !intel_num_dvo_pipes(dev_priv)) {
1577                 I915_WRITE(DPLL(PIPE_B),
1578                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1579                 I915_WRITE(DPLL(PIPE_A),
1580                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1581         }
1582
1583         /* Don't disable pipe or pipe PLLs if needed */
1584         if (IS_I830(dev_priv))
1585                 return;
1586
1587         /* Make sure the pipe isn't still relying on us */
1588         assert_pipe_disabled(dev_priv, pipe);
1589
1590         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1591         POSTING_READ(DPLL(pipe));
1592 }
1593
1594 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1595 {
1596         u32 val;
1597
1598         /* Make sure the pipe isn't still relying on us */
1599         assert_pipe_disabled(dev_priv, pipe);
1600
1601         val = DPLL_INTEGRATED_REF_CLK_VLV |
1602                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1603         if (pipe != PIPE_A)
1604                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1605
1606         I915_WRITE(DPLL(pipe), val);
1607         POSTING_READ(DPLL(pipe));
1608 }
1609
1610 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1611 {
1612         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1613         u32 val;
1614
1615         /* Make sure the pipe isn't still relying on us */
1616         assert_pipe_disabled(dev_priv, pipe);
1617
1618         val = DPLL_SSC_REF_CLK_CHV |
1619                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1620         if (pipe != PIPE_A)
1621                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1622
1623         I915_WRITE(DPLL(pipe), val);
1624         POSTING_READ(DPLL(pipe));
1625
1626         mutex_lock(&dev_priv->sb_lock);
1627
1628         /* Disable 10bit clock to display controller */
1629         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1630         val &= ~DPIO_DCLKP_EN;
1631         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1632
1633         mutex_unlock(&dev_priv->sb_lock);
1634 }
1635
1636 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1637                          struct intel_digital_port *dport,
1638                          unsigned int expected_mask)
1639 {
1640         u32 port_mask;
1641         i915_reg_t dpll_reg;
1642
1643         switch (dport->base.port) {
1644         case PORT_B:
1645                 port_mask = DPLL_PORTB_READY_MASK;
1646                 dpll_reg = DPLL(0);
1647                 break;
1648         case PORT_C:
1649                 port_mask = DPLL_PORTC_READY_MASK;
1650                 dpll_reg = DPLL(0);
1651                 expected_mask <<= 4;
1652                 break;
1653         case PORT_D:
1654                 port_mask = DPLL_PORTD_READY_MASK;
1655                 dpll_reg = DPIO_PHY_STATUS;
1656                 break;
1657         default:
1658                 BUG();
1659         }
1660
1661         if (intel_wait_for_register(dev_priv,
1662                                     dpll_reg, port_mask, expected_mask,
1663                                     1000))
1664                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1665                      port_name(dport->base.port),
1666                      I915_READ(dpll_reg) & port_mask, expected_mask);
1667 }
1668
1669 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1670                                            enum pipe pipe)
1671 {
1672         struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
1673                                                                 pipe);
1674         i915_reg_t reg;
1675         uint32_t val, pipeconf_val;
1676
1677         /* Make sure PCH DPLL is enabled */
1678         assert_shared_dpll_enabled(dev_priv, intel_crtc->config->shared_dpll);
1679
1680         /* FDI must be feeding us bits for PCH ports */
1681         assert_fdi_tx_enabled(dev_priv, pipe);
1682         assert_fdi_rx_enabled(dev_priv, pipe);
1683
1684         if (HAS_PCH_CPT(dev_priv)) {
1685                 /* Workaround: Set the timing override bit before enabling the
1686                  * pch transcoder. */
1687                 reg = TRANS_CHICKEN2(pipe);
1688                 val = I915_READ(reg);
1689                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1690                 I915_WRITE(reg, val);
1691         }
1692
1693         reg = PCH_TRANSCONF(pipe);
1694         val = I915_READ(reg);
1695         pipeconf_val = I915_READ(PIPECONF(pipe));
1696
1697         if (HAS_PCH_IBX(dev_priv)) {
1698                 /*
1699                  * Make the BPC in transcoder be consistent with
1700                  * that in pipeconf reg. For HDMI we must use 8bpc
1701                  * here for both 8bpc and 12bpc.
1702                  */
1703                 val &= ~PIPECONF_BPC_MASK;
1704                 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_HDMI))
1705                         val |= PIPECONF_8BPC;
1706                 else
1707                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1708         }
1709
1710         val &= ~TRANS_INTERLACE_MASK;
1711         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1712                 if (HAS_PCH_IBX(dev_priv) &&
1713                     intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
1714                         val |= TRANS_LEGACY_INTERLACED_ILK;
1715                 else
1716                         val |= TRANS_INTERLACED;
1717         else
1718                 val |= TRANS_PROGRESSIVE;
1719
1720         I915_WRITE(reg, val | TRANS_ENABLE);
1721         if (intel_wait_for_register(dev_priv,
1722                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1723                                     100))
1724                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1725 }
1726
1727 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1728                                       enum transcoder cpu_transcoder)
1729 {
1730         u32 val, pipeconf_val;
1731
1732         /* FDI must be feeding us bits for PCH ports */
1733         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1734         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1735
1736         /* Workaround: set timing override bit. */
1737         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1738         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1739         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1740
1741         val = TRANS_ENABLE;
1742         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1743
1744         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1745             PIPECONF_INTERLACED_ILK)
1746                 val |= TRANS_INTERLACED;
1747         else
1748                 val |= TRANS_PROGRESSIVE;
1749
1750         I915_WRITE(LPT_TRANSCONF, val);
1751         if (intel_wait_for_register(dev_priv,
1752                                     LPT_TRANSCONF,
1753                                     TRANS_STATE_ENABLE,
1754                                     TRANS_STATE_ENABLE,
1755                                     100))
1756                 DRM_ERROR("Failed to enable PCH transcoder\n");
1757 }
1758
1759 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1760                                             enum pipe pipe)
1761 {
1762         i915_reg_t reg;
1763         uint32_t val;
1764
1765         /* FDI relies on the transcoder */
1766         assert_fdi_tx_disabled(dev_priv, pipe);
1767         assert_fdi_rx_disabled(dev_priv, pipe);
1768
1769         /* Ports must be off as well */
1770         assert_pch_ports_disabled(dev_priv, pipe);
1771
1772         reg = PCH_TRANSCONF(pipe);
1773         val = I915_READ(reg);
1774         val &= ~TRANS_ENABLE;
1775         I915_WRITE(reg, val);
1776         /* wait for PCH transcoder off, transcoder state */
1777         if (intel_wait_for_register(dev_priv,
1778                                     reg, TRANS_STATE_ENABLE, 0,
1779                                     50))
1780                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1781
1782         if (HAS_PCH_CPT(dev_priv)) {
1783                 /* Workaround: Clear the timing override chicken bit again. */
1784                 reg = TRANS_CHICKEN2(pipe);
1785                 val = I915_READ(reg);
1786                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1787                 I915_WRITE(reg, val);
1788         }
1789 }
1790
1791 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1792 {
1793         u32 val;
1794
1795         val = I915_READ(LPT_TRANSCONF);
1796         val &= ~TRANS_ENABLE;
1797         I915_WRITE(LPT_TRANSCONF, val);
1798         /* wait for PCH transcoder off, transcoder state */
1799         if (intel_wait_for_register(dev_priv,
1800                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1801                                     50))
1802                 DRM_ERROR("Failed to disable PCH transcoder\n");
1803
1804         /* Workaround: clear timing override bit. */
1805         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1806         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1807         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1808 }
1809
1810 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1811 {
1812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1813
1814         if (HAS_PCH_LPT(dev_priv))
1815                 return PIPE_A;
1816         else
1817                 return crtc->pipe;
1818 }
1819
1820 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1821 {
1822         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1823         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1824         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1825         enum pipe pipe = crtc->pipe;
1826         i915_reg_t reg;
1827         u32 val;
1828
1829         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1830
1831         assert_planes_disabled(crtc);
1832
1833         /*
1834          * A pipe without a PLL won't actually be able to drive bits from
1835          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1836          * need the check.
1837          */
1838         if (HAS_GMCH_DISPLAY(dev_priv)) {
1839                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1840                         assert_dsi_pll_enabled(dev_priv);
1841                 else
1842                         assert_pll_enabled(dev_priv, pipe);
1843         } else {
1844                 if (new_crtc_state->has_pch_encoder) {
1845                         /* if driving the PCH, we need FDI enabled */
1846                         assert_fdi_rx_pll_enabled(dev_priv,
1847                                                   intel_crtc_pch_transcoder(crtc));
1848                         assert_fdi_tx_pll_enabled(dev_priv,
1849                                                   (enum pipe) cpu_transcoder);
1850                 }
1851                 /* FIXME: assert CPU port conditions for SNB+ */
1852         }
1853
1854         reg = PIPECONF(cpu_transcoder);
1855         val = I915_READ(reg);
1856         if (val & PIPECONF_ENABLE) {
1857                 /* we keep both pipes enabled on 830 */
1858                 WARN_ON(!IS_I830(dev_priv));
1859                 return;
1860         }
1861
1862         I915_WRITE(reg, val | PIPECONF_ENABLE);
1863         POSTING_READ(reg);
1864
1865         /*
1866          * Until the pipe starts PIPEDSL reads will return a stale value,
1867          * which causes an apparent vblank timestamp jump when PIPEDSL
1868          * resets to its proper value. That also messes up the frame count
1869          * when it's derived from the timestamps. So let's wait for the
1870          * pipe to start properly before we call drm_crtc_vblank_on()
1871          */
1872         if (dev_priv->drm.max_vblank_count == 0)
1873                 intel_wait_for_pipe_scanline_moving(crtc);
1874 }
1875
1876 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1877 {
1878         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1879         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1880         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1881         enum pipe pipe = crtc->pipe;
1882         i915_reg_t reg;
1883         u32 val;
1884
1885         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1886
1887         /*
1888          * Make sure planes won't keep trying to pump pixels to us,
1889          * or we might hang the display.
1890          */
1891         assert_planes_disabled(crtc);
1892
1893         reg = PIPECONF(cpu_transcoder);
1894         val = I915_READ(reg);
1895         if ((val & PIPECONF_ENABLE) == 0)
1896                 return;
1897
1898         /*
1899          * Double wide has implications for planes
1900          * so best keep it disabled when not needed.
1901          */
1902         if (old_crtc_state->double_wide)
1903                 val &= ~PIPECONF_DOUBLE_WIDE;
1904
1905         /* Don't disable pipe or pipe PLLs if needed */
1906         if (!IS_I830(dev_priv))
1907                 val &= ~PIPECONF_ENABLE;
1908
1909         I915_WRITE(reg, val);
1910         if ((val & PIPECONF_ENABLE) == 0)
1911                 intel_wait_for_pipe_off(old_crtc_state);
1912 }
1913
1914 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1915 {
1916         return IS_GEN2(dev_priv) ? 2048 : 4096;
1917 }
1918
1919 static unsigned int
1920 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1921 {
1922         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1923         unsigned int cpp = fb->format->cpp[color_plane];
1924
1925         switch (fb->modifier) {
1926         case DRM_FORMAT_MOD_LINEAR:
1927                 return cpp;
1928         case I915_FORMAT_MOD_X_TILED:
1929                 if (IS_GEN2(dev_priv))
1930                         return 128;
1931                 else
1932                         return 512;
1933         case I915_FORMAT_MOD_Y_TILED_CCS:
1934                 if (color_plane == 1)
1935                         return 128;
1936                 /* fall through */
1937         case I915_FORMAT_MOD_Y_TILED:
1938                 if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv))
1939                         return 128;
1940                 else
1941                         return 512;
1942         case I915_FORMAT_MOD_Yf_TILED_CCS:
1943                 if (color_plane == 1)
1944                         return 128;
1945                 /* fall through */
1946         case I915_FORMAT_MOD_Yf_TILED:
1947                 switch (cpp) {
1948                 case 1:
1949                         return 64;
1950                 case 2:
1951                 case 4:
1952                         return 128;
1953                 case 8:
1954                 case 16:
1955                         return 256;
1956                 default:
1957                         MISSING_CASE(cpp);
1958                         return cpp;
1959                 }
1960                 break;
1961         default:
1962                 MISSING_CASE(fb->modifier);
1963                 return cpp;
1964         }
1965 }
1966
1967 static unsigned int
1968 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1969 {
1970         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
1971                 return 1;
1972         else
1973                 return intel_tile_size(to_i915(fb->dev)) /
1974                         intel_tile_width_bytes(fb, color_plane);
1975 }
1976
1977 /* Return the tile dimensions in pixel units */
1978 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1979                             unsigned int *tile_width,
1980                             unsigned int *tile_height)
1981 {
1982         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1983         unsigned int cpp = fb->format->cpp[color_plane];
1984
1985         *tile_width = tile_width_bytes / cpp;
1986         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1987 }
1988
1989 unsigned int
1990 intel_fb_align_height(const struct drm_framebuffer *fb,
1991                       int color_plane, unsigned int height)
1992 {
1993         unsigned int tile_height = intel_tile_height(fb, color_plane);
1994
1995         return ALIGN(height, tile_height);
1996 }
1997
1998 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1999 {
2000         unsigned int size = 0;
2001         int i;
2002
2003         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2004                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2005
2006         return size;
2007 }
2008
2009 static void
2010 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2011                         const struct drm_framebuffer *fb,
2012                         unsigned int rotation)
2013 {
2014         view->type = I915_GGTT_VIEW_NORMAL;
2015         if (drm_rotation_90_or_270(rotation)) {
2016                 view->type = I915_GGTT_VIEW_ROTATED;
2017                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2018         }
2019 }
2020
2021 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2022 {
2023         if (IS_I830(dev_priv))
2024                 return 16 * 1024;
2025         else if (IS_I85X(dev_priv))
2026                 return 256;
2027         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2028                 return 32;
2029         else
2030                 return 4 * 1024;
2031 }
2032
2033 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2034 {
2035         if (INTEL_GEN(dev_priv) >= 9)
2036                 return 256 * 1024;
2037         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2038                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2039                 return 128 * 1024;
2040         else if (INTEL_GEN(dev_priv) >= 4)
2041                 return 4 * 1024;
2042         else
2043                 return 0;
2044 }
2045
2046 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2047                                          int color_plane)
2048 {
2049         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2050
2051         /* AUX_DIST needs only 4K alignment */
2052         if (color_plane == 1)
2053                 return 4096;
2054
2055         switch (fb->modifier) {
2056         case DRM_FORMAT_MOD_LINEAR:
2057                 return intel_linear_alignment(dev_priv);
2058         case I915_FORMAT_MOD_X_TILED:
2059                 if (INTEL_GEN(dev_priv) >= 9)
2060                         return 256 * 1024;
2061                 return 0;
2062         case I915_FORMAT_MOD_Y_TILED_CCS:
2063         case I915_FORMAT_MOD_Yf_TILED_CCS:
2064         case I915_FORMAT_MOD_Y_TILED:
2065         case I915_FORMAT_MOD_Yf_TILED:
2066                 return 1 * 1024 * 1024;
2067         default:
2068                 MISSING_CASE(fb->modifier);
2069                 return 0;
2070         }
2071 }
2072
2073 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2074 {
2075         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2076         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2077
2078         return INTEL_GEN(dev_priv) < 4 || plane->has_fbc;
2079 }
2080
2081 struct i915_vma *
2082 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2083                            const struct i915_ggtt_view *view,
2084                            bool uses_fence,
2085                            unsigned long *out_flags)
2086 {
2087         struct drm_device *dev = fb->dev;
2088         struct drm_i915_private *dev_priv = to_i915(dev);
2089         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2090         struct i915_vma *vma;
2091         unsigned int pinctl;
2092         u32 alignment;
2093
2094         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2095
2096         alignment = intel_surf_alignment(fb, 0);
2097
2098         /* Note that the w/a also requires 64 PTE of padding following the
2099          * bo. We currently fill all unused PTE with the shadow page and so
2100          * we should always have valid PTE following the scanout preventing
2101          * the VT-d warning.
2102          */
2103         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2104                 alignment = 256 * 1024;
2105
2106         /*
2107          * Global gtt pte registers are special registers which actually forward
2108          * writes to a chunk of system memory. Which means that there is no risk
2109          * that the register values disappear as soon as we call
2110          * intel_runtime_pm_put(), so it is correct to wrap only the
2111          * pin/unpin/fence and not more.
2112          */
2113         intel_runtime_pm_get(dev_priv);
2114
2115         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2116
2117         pinctl = 0;
2118
2119         /* Valleyview is definitely limited to scanning out the first
2120          * 512MiB. Lets presume this behaviour was inherited from the
2121          * g4x display engine and that all earlier gen are similarly
2122          * limited. Testing suggests that it is a little more
2123          * complicated than this. For example, Cherryview appears quite
2124          * happy to scanout from anywhere within its global aperture.
2125          */
2126         if (HAS_GMCH_DISPLAY(dev_priv))
2127                 pinctl |= PIN_MAPPABLE;
2128
2129         vma = i915_gem_object_pin_to_display_plane(obj,
2130                                                    alignment, view, pinctl);
2131         if (IS_ERR(vma))
2132                 goto err;
2133
2134         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2135                 int ret;
2136
2137                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2138                  * fence, whereas 965+ only requires a fence if using
2139                  * framebuffer compression.  For simplicity, we always, when
2140                  * possible, install a fence as the cost is not that onerous.
2141                  *
2142                  * If we fail to fence the tiled scanout, then either the
2143                  * modeset will reject the change (which is highly unlikely as
2144                  * the affected systems, all but one, do not have unmappable
2145                  * space) or we will not be able to enable full powersaving
2146                  * techniques (also likely not to apply due to various limits
2147                  * FBC and the like impose on the size of the buffer, which
2148                  * presumably we violated anyway with this unmappable buffer).
2149                  * Anyway, it is presumably better to stumble onwards with
2150                  * something and try to run the system in a "less than optimal"
2151                  * mode that matches the user configuration.
2152                  */
2153                 ret = i915_vma_pin_fence(vma);
2154                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2155                         i915_gem_object_unpin_from_display_plane(vma);
2156                         vma = ERR_PTR(ret);
2157                         goto err;
2158                 }
2159
2160                 if (ret == 0 && vma->fence)
2161                         *out_flags |= PLANE_HAS_FENCE;
2162         }
2163
2164         i915_vma_get(vma);
2165 err:
2166         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2167
2168         intel_runtime_pm_put(dev_priv);
2169         return vma;
2170 }
2171
2172 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2173 {
2174         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2175
2176         if (flags & PLANE_HAS_FENCE)
2177                 i915_vma_unpin_fence(vma);
2178         i915_gem_object_unpin_from_display_plane(vma);
2179         i915_vma_put(vma);
2180 }
2181
2182 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2183                           unsigned int rotation)
2184 {
2185         if (drm_rotation_90_or_270(rotation))
2186                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2187         else
2188                 return fb->pitches[color_plane];
2189 }
2190
2191 /*
2192  * Convert the x/y offsets into a linear offset.
2193  * Only valid with 0/180 degree rotation, which is fine since linear
2194  * offset is only used with linear buffers on pre-hsw and tiled buffers
2195  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2196  */
2197 u32 intel_fb_xy_to_linear(int x, int y,
2198                           const struct intel_plane_state *state,
2199                           int color_plane)
2200 {
2201         const struct drm_framebuffer *fb = state->base.fb;
2202         unsigned int cpp = fb->format->cpp[color_plane];
2203         unsigned int pitch = state->color_plane[color_plane].stride;
2204
2205         return y * pitch + x * cpp;
2206 }
2207
2208 /*
2209  * Add the x/y offsets derived from fb->offsets[] to the user
2210  * specified plane src x/y offsets. The resulting x/y offsets
2211  * specify the start of scanout from the beginning of the gtt mapping.
2212  */
2213 void intel_add_fb_offsets(int *x, int *y,
2214                           const struct intel_plane_state *state,
2215                           int color_plane)
2216
2217 {
2218         const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
2219         unsigned int rotation = state->base.rotation;
2220
2221         if (drm_rotation_90_or_270(rotation)) {
2222                 *x += intel_fb->rotated[color_plane].x;
2223                 *y += intel_fb->rotated[color_plane].y;
2224         } else {
2225                 *x += intel_fb->normal[color_plane].x;
2226                 *y += intel_fb->normal[color_plane].y;
2227         }
2228 }
2229
2230 static u32 intel_adjust_tile_offset(int *x, int *y,
2231                                     unsigned int tile_width,
2232                                     unsigned int tile_height,
2233                                     unsigned int tile_size,
2234                                     unsigned int pitch_tiles,
2235                                     u32 old_offset,
2236                                     u32 new_offset)
2237 {
2238         unsigned int pitch_pixels = pitch_tiles * tile_width;
2239         unsigned int tiles;
2240
2241         WARN_ON(old_offset & (tile_size - 1));
2242         WARN_ON(new_offset & (tile_size - 1));
2243         WARN_ON(new_offset > old_offset);
2244
2245         tiles = (old_offset - new_offset) / tile_size;
2246
2247         *y += tiles / pitch_tiles * tile_height;
2248         *x += tiles % pitch_tiles * tile_width;
2249
2250         /* minimize x in case it got needlessly big */
2251         *y += *x / pitch_pixels * tile_height;
2252         *x %= pitch_pixels;
2253
2254         return new_offset;
2255 }
2256
2257 static u32 intel_adjust_aligned_offset(int *x, int *y,
2258                                        const struct drm_framebuffer *fb,
2259                                        int color_plane,
2260                                        unsigned int rotation,
2261                                        unsigned int pitch,
2262                                        u32 old_offset, u32 new_offset)
2263 {
2264         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2265         unsigned int cpp = fb->format->cpp[color_plane];
2266
2267         WARN_ON(new_offset > old_offset);
2268
2269         if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
2270                 unsigned int tile_size, tile_width, tile_height;
2271                 unsigned int pitch_tiles;
2272
2273                 tile_size = intel_tile_size(dev_priv);
2274                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2275
2276                 if (drm_rotation_90_or_270(rotation)) {
2277                         pitch_tiles = pitch / tile_height;
2278                         swap(tile_width, tile_height);
2279                 } else {
2280                         pitch_tiles = pitch / (tile_width * cpp);
2281                 }
2282
2283                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2284                                          tile_size, pitch_tiles,
2285                                          old_offset, new_offset);
2286         } else {
2287                 old_offset += *y * pitch + *x * cpp;
2288
2289                 *y = (old_offset - new_offset) / pitch;
2290                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2291         }
2292
2293         return new_offset;
2294 }
2295
2296 /*
2297  * Adjust the tile offset by moving the difference into
2298  * the x/y offsets.
2299  */
2300 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2301                                              const struct intel_plane_state *state,
2302                                              int color_plane,
2303                                              u32 old_offset, u32 new_offset)
2304 {
2305         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2306                                            state->base.rotation,
2307                                            state->color_plane[color_plane].stride,
2308                                            old_offset, new_offset);
2309 }
2310
2311 /*
2312  * Computes the aligned offset to the base tile and adjusts
2313  * x, y. bytes per pixel is assumed to be a power-of-two.
2314  *
2315  * In the 90/270 rotated case, x and y are assumed
2316  * to be already rotated to match the rotated GTT view, and
2317  * pitch is the tile_height aligned framebuffer height.
2318  *
2319  * This function is used when computing the derived information
2320  * under intel_framebuffer, so using any of that information
2321  * here is not allowed. Anything under drm_framebuffer can be
2322  * used. This is why the user has to pass in the pitch since it
2323  * is specified in the rotated orientation.
2324  */
2325 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2326                                         int *x, int *y,
2327                                         const struct drm_framebuffer *fb,
2328                                         int color_plane,
2329                                         unsigned int pitch,
2330                                         unsigned int rotation,
2331                                         u32 alignment)
2332 {
2333         uint64_t fb_modifier = fb->modifier;
2334         unsigned int cpp = fb->format->cpp[color_plane];
2335         u32 offset, offset_aligned;
2336
2337         if (alignment)
2338                 alignment--;
2339
2340         if (fb_modifier != DRM_FORMAT_MOD_LINEAR) {
2341                 unsigned int tile_size, tile_width, tile_height;
2342                 unsigned int tile_rows, tiles, pitch_tiles;
2343
2344                 tile_size = intel_tile_size(dev_priv);
2345                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2346
2347                 if (drm_rotation_90_or_270(rotation)) {
2348                         pitch_tiles = pitch / tile_height;
2349                         swap(tile_width, tile_height);
2350                 } else {
2351                         pitch_tiles = pitch / (tile_width * cpp);
2352                 }
2353
2354                 tile_rows = *y / tile_height;
2355                 *y %= tile_height;
2356
2357                 tiles = *x / tile_width;
2358                 *x %= tile_width;
2359
2360                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2361                 offset_aligned = offset & ~alignment;
2362
2363                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2364                                          tile_size, pitch_tiles,
2365                                          offset, offset_aligned);
2366         } else {
2367                 offset = *y * pitch + *x * cpp;
2368                 offset_aligned = offset & ~alignment;
2369
2370                 *y = (offset & alignment) / pitch;
2371                 *x = ((offset & alignment) - *y * pitch) / cpp;
2372         }
2373
2374         return offset_aligned;
2375 }
2376
2377 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2378                                               const struct intel_plane_state *state,
2379                                               int color_plane)
2380 {
2381         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2382         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2383         const struct drm_framebuffer *fb = state->base.fb;
2384         unsigned int rotation = state->base.rotation;
2385         int pitch = state->color_plane[color_plane].stride;
2386         u32 alignment;
2387
2388         if (intel_plane->id == PLANE_CURSOR)
2389                 alignment = intel_cursor_alignment(dev_priv);
2390         else
2391                 alignment = intel_surf_alignment(fb, color_plane);
2392
2393         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2394                                             pitch, rotation, alignment);
2395 }
2396
2397 /* Convert the fb->offset[] into x/y offsets */
2398 static int intel_fb_offset_to_xy(int *x, int *y,
2399                                  const struct drm_framebuffer *fb,
2400                                  int color_plane)
2401 {
2402         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2403
2404         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2405             fb->offsets[color_plane] % intel_tile_size(dev_priv))
2406                 return -EINVAL;
2407
2408         *x = 0;
2409         *y = 0;
2410
2411         intel_adjust_aligned_offset(x, y,
2412                                     fb, color_plane, DRM_MODE_ROTATE_0,
2413                                     fb->pitches[color_plane],
2414                                     fb->offsets[color_plane], 0);
2415
2416         return 0;
2417 }
2418
2419 static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
2420 {
2421         switch (fb_modifier) {
2422         case I915_FORMAT_MOD_X_TILED:
2423                 return I915_TILING_X;
2424         case I915_FORMAT_MOD_Y_TILED:
2425         case I915_FORMAT_MOD_Y_TILED_CCS:
2426                 return I915_TILING_Y;
2427         default:
2428                 return I915_TILING_NONE;
2429         }
2430 }
2431
2432 /*
2433  * From the Sky Lake PRM:
2434  * "The Color Control Surface (CCS) contains the compression status of
2435  *  the cache-line pairs. The compression state of the cache-line pair
2436  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2437  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2438  *  cache-line-pairs. CCS is always Y tiled."
2439  *
2440  * Since cache line pairs refers to horizontally adjacent cache lines,
2441  * each cache line in the CCS corresponds to an area of 32x16 cache
2442  * lines on the main surface. Since each pixel is 4 bytes, this gives
2443  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2444  * main surface.
2445  */
2446 static const struct drm_format_info ccs_formats[] = {
2447         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2448         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2449         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2450         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2451 };
2452
2453 static const struct drm_format_info *
2454 lookup_format_info(const struct drm_format_info formats[],
2455                    int num_formats, u32 format)
2456 {
2457         int i;
2458
2459         for (i = 0; i < num_formats; i++) {
2460                 if (formats[i].format == format)
2461                         return &formats[i];
2462         }
2463
2464         return NULL;
2465 }
2466
2467 static const struct drm_format_info *
2468 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2469 {
2470         switch (cmd->modifier[0]) {
2471         case I915_FORMAT_MOD_Y_TILED_CCS:
2472         case I915_FORMAT_MOD_Yf_TILED_CCS:
2473                 return lookup_format_info(ccs_formats,
2474                                           ARRAY_SIZE(ccs_formats),
2475                                           cmd->pixel_format);
2476         default:
2477                 return NULL;
2478         }
2479 }
2480
2481 bool is_ccs_modifier(u64 modifier)
2482 {
2483         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2484                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2485 }
2486
2487 static int
2488 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2489                    struct drm_framebuffer *fb)
2490 {
2491         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2492         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2493         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2494         u32 gtt_offset_rotated = 0;
2495         unsigned int max_size = 0;
2496         int i, num_planes = fb->format->num_planes;
2497         unsigned int tile_size = intel_tile_size(dev_priv);
2498
2499         for (i = 0; i < num_planes; i++) {
2500                 unsigned int width, height;
2501                 unsigned int cpp, size;
2502                 u32 offset;
2503                 int x, y;
2504                 int ret;
2505
2506                 cpp = fb->format->cpp[i];
2507                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2508                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2509
2510                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2511                 if (ret) {
2512                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2513                                       i, fb->offsets[i]);
2514                         return ret;
2515                 }
2516
2517                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2518                         int hsub = fb->format->hsub;
2519                         int vsub = fb->format->vsub;
2520                         int tile_width, tile_height;
2521                         int main_x, main_y;
2522                         int ccs_x, ccs_y;
2523
2524                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2525                         tile_width *= hsub;
2526                         tile_height *= vsub;
2527
2528                         ccs_x = (x * hsub) % tile_width;
2529                         ccs_y = (y * vsub) % tile_height;
2530                         main_x = intel_fb->normal[0].x % tile_width;
2531                         main_y = intel_fb->normal[0].y % tile_height;
2532
2533                         /*
2534                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2535                          * x/y offsets must match between CCS and the main surface.
2536                          */
2537                         if (main_x != ccs_x || main_y != ccs_y) {
2538                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2539                                               main_x, main_y,
2540                                               ccs_x, ccs_y,
2541                                               intel_fb->normal[0].x,
2542                                               intel_fb->normal[0].y,
2543                                               x, y);
2544                                 return -EINVAL;
2545                         }
2546                 }
2547
2548                 /*
2549                  * The fence (if used) is aligned to the start of the object
2550                  * so having the framebuffer wrap around across the edge of the
2551                  * fenced region doesn't really work. We have no API to configure
2552                  * the fence start offset within the object (nor could we probably
2553                  * on gen2/3). So it's just easier if we just require that the
2554                  * fb layout agrees with the fence layout. We already check that the
2555                  * fb stride matches the fence stride elsewhere.
2556                  */
2557                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2558                     (x + width) * cpp > fb->pitches[i]) {
2559                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2560                                       i, fb->offsets[i]);
2561                         return -EINVAL;
2562                 }
2563
2564                 /*
2565                  * First pixel of the framebuffer from
2566                  * the start of the normal gtt mapping.
2567                  */
2568                 intel_fb->normal[i].x = x;
2569                 intel_fb->normal[i].y = y;
2570
2571                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2572                                                       fb->pitches[i],
2573                                                       DRM_MODE_ROTATE_0,
2574                                                       tile_size);
2575                 offset /= tile_size;
2576
2577                 if (fb->modifier != DRM_FORMAT_MOD_LINEAR) {
2578                         unsigned int tile_width, tile_height;
2579                         unsigned int pitch_tiles;
2580                         struct drm_rect r;
2581
2582                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2583
2584                         rot_info->plane[i].offset = offset;
2585                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2586                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2587                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2588
2589                         intel_fb->rotated[i].pitch =
2590                                 rot_info->plane[i].height * tile_height;
2591
2592                         /* how many tiles does this plane need */
2593                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2594                         /*
2595                          * If the plane isn't horizontally tile aligned,
2596                          * we need one more tile.
2597                          */
2598                         if (x != 0)
2599                                 size++;
2600
2601                         /* rotate the x/y offsets to match the GTT view */
2602                         r.x1 = x;
2603                         r.y1 = y;
2604                         r.x2 = x + width;
2605                         r.y2 = y + height;
2606                         drm_rect_rotate(&r,
2607                                         rot_info->plane[i].width * tile_width,
2608                                         rot_info->plane[i].height * tile_height,
2609                                         DRM_MODE_ROTATE_270);
2610                         x = r.x1;
2611                         y = r.y1;
2612
2613                         /* rotate the tile dimensions to match the GTT view */
2614                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2615                         swap(tile_width, tile_height);
2616
2617                         /*
2618                          * We only keep the x/y offsets, so push all of the
2619                          * gtt offset into the x/y offsets.
2620                          */
2621                         intel_adjust_tile_offset(&x, &y,
2622                                                  tile_width, tile_height,
2623                                                  tile_size, pitch_tiles,
2624                                                  gtt_offset_rotated * tile_size, 0);
2625
2626                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2627
2628                         /*
2629                          * First pixel of the framebuffer from
2630                          * the start of the rotated gtt mapping.
2631                          */
2632                         intel_fb->rotated[i].x = x;
2633                         intel_fb->rotated[i].y = y;
2634                 } else {
2635                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2636                                             x * cpp, tile_size);
2637                 }
2638
2639                 /* how many tiles in total needed in the bo */
2640                 max_size = max(max_size, offset + size);
2641         }
2642
2643         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2644                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2645                               mul_u32_u32(max_size, tile_size), obj->base.size);
2646                 return -EINVAL;
2647         }
2648
2649         return 0;
2650 }
2651
2652 static int i9xx_format_to_fourcc(int format)
2653 {
2654         switch (format) {
2655         case DISPPLANE_8BPP:
2656                 return DRM_FORMAT_C8;
2657         case DISPPLANE_BGRX555:
2658                 return DRM_FORMAT_XRGB1555;
2659         case DISPPLANE_BGRX565:
2660                 return DRM_FORMAT_RGB565;
2661         default:
2662         case DISPPLANE_BGRX888:
2663                 return DRM_FORMAT_XRGB8888;
2664         case DISPPLANE_RGBX888:
2665                 return DRM_FORMAT_XBGR8888;
2666         case DISPPLANE_BGRX101010:
2667                 return DRM_FORMAT_XRGB2101010;
2668         case DISPPLANE_RGBX101010:
2669                 return DRM_FORMAT_XBGR2101010;
2670         }
2671 }
2672
2673 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2674 {
2675         switch (format) {
2676         case PLANE_CTL_FORMAT_RGB_565:
2677                 return DRM_FORMAT_RGB565;
2678         case PLANE_CTL_FORMAT_NV12:
2679                 return DRM_FORMAT_NV12;
2680         default:
2681         case PLANE_CTL_FORMAT_XRGB_8888:
2682                 if (rgb_order) {
2683                         if (alpha)
2684                                 return DRM_FORMAT_ABGR8888;
2685                         else
2686                                 return DRM_FORMAT_XBGR8888;
2687                 } else {
2688                         if (alpha)
2689                                 return DRM_FORMAT_ARGB8888;
2690                         else
2691                                 return DRM_FORMAT_XRGB8888;
2692                 }
2693         case PLANE_CTL_FORMAT_XRGB_2101010:
2694                 if (rgb_order)
2695                         return DRM_FORMAT_XBGR2101010;
2696                 else
2697                         return DRM_FORMAT_XRGB2101010;
2698         }
2699 }
2700
2701 static bool
2702 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2703                               struct intel_initial_plane_config *plane_config)
2704 {
2705         struct drm_device *dev = crtc->base.dev;
2706         struct drm_i915_private *dev_priv = to_i915(dev);
2707         struct drm_i915_gem_object *obj = NULL;
2708         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2709         struct drm_framebuffer *fb = &plane_config->fb->base;
2710         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2711         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2712                                     PAGE_SIZE);
2713
2714         size_aligned -= base_aligned;
2715
2716         if (plane_config->size == 0)
2717                 return false;
2718
2719         /* If the FB is too big, just don't use it since fbdev is not very
2720          * important and we should probably use that space with FBC or other
2721          * features. */
2722         if (size_aligned * 2 > dev_priv->stolen_usable_size)
2723                 return false;
2724
2725         mutex_lock(&dev->struct_mutex);
2726         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
2727                                                              base_aligned,
2728                                                              base_aligned,
2729                                                              size_aligned);
2730         mutex_unlock(&dev->struct_mutex);
2731         if (!obj)
2732                 return false;
2733
2734         if (plane_config->tiling == I915_TILING_X)
2735                 obj->tiling_and_stride = fb->pitches[0] | I915_TILING_X;
2736
2737         mode_cmd.pixel_format = fb->format->format;
2738         mode_cmd.width = fb->width;
2739         mode_cmd.height = fb->height;
2740         mode_cmd.pitches[0] = fb->pitches[0];
2741         mode_cmd.modifier[0] = fb->modifier;
2742         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2743
2744         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
2745                 DRM_DEBUG_KMS("intel fb init failed\n");
2746                 goto out_unref_obj;
2747         }
2748
2749
2750         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2751         return true;
2752
2753 out_unref_obj:
2754         i915_gem_object_put(obj);
2755         return false;
2756 }
2757
2758 static void
2759 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
2760                         struct intel_plane_state *plane_state,
2761                         bool visible)
2762 {
2763         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2764
2765         plane_state->base.visible = visible;
2766
2767         if (visible)
2768                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
2769         else
2770                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
2771 }
2772
2773 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
2774 {
2775         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2776         struct drm_plane *plane;
2777
2778         /*
2779          * Active_planes aliases if multiple "primary" or cursor planes
2780          * have been used on the same (or wrong) pipe. plane_mask uses
2781          * unique ids, hence we can use that to reconstruct active_planes.
2782          */
2783         crtc_state->active_planes = 0;
2784
2785         drm_for_each_plane_mask(plane, &dev_priv->drm,
2786                                 crtc_state->base.plane_mask)
2787                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
2788 }
2789
2790 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
2791                                          struct intel_plane *plane)
2792 {
2793         struct intel_crtc_state *crtc_state =
2794                 to_intel_crtc_state(crtc->base.state);
2795         struct intel_plane_state *plane_state =
2796                 to_intel_plane_state(plane->base.state);
2797
2798         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
2799                       plane->base.base.id, plane->base.name,
2800                       crtc->base.base.id, crtc->base.name);
2801
2802         intel_set_plane_visible(crtc_state, plane_state, false);
2803         fixup_active_planes(crtc_state);
2804
2805         if (plane->id == PLANE_PRIMARY)
2806                 intel_pre_disable_primary_noatomic(&crtc->base);
2807
2808         trace_intel_disable_plane(&plane->base, crtc);
2809         plane->disable_plane(plane, crtc);
2810 }
2811
2812 static void
2813 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2814                              struct intel_initial_plane_config *plane_config)
2815 {
2816         struct drm_device *dev = intel_crtc->base.dev;
2817         struct drm_i915_private *dev_priv = to_i915(dev);
2818         struct drm_crtc *c;
2819         struct drm_i915_gem_object *obj;
2820         struct drm_plane *primary = intel_crtc->base.primary;
2821         struct drm_plane_state *plane_state = primary->state;
2822         struct intel_plane *intel_plane = to_intel_plane(primary);
2823         struct intel_plane_state *intel_state =
2824                 to_intel_plane_state(plane_state);
2825         struct drm_framebuffer *fb;
2826
2827         if (!plane_config->fb)
2828                 return;
2829
2830         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2831                 fb = &plane_config->fb->base;
2832                 goto valid_fb;
2833         }
2834
2835         kfree(plane_config->fb);
2836
2837         /*
2838          * Failed to alloc the obj, check to see if we should share
2839          * an fb with another CRTC instead
2840          */
2841         for_each_crtc(dev, c) {
2842                 struct intel_plane_state *state;
2843
2844                 if (c == &intel_crtc->base)
2845                         continue;
2846
2847                 if (!to_intel_crtc(c)->active)
2848                         continue;
2849
2850                 state = to_intel_plane_state(c->primary->state);
2851                 if (!state->vma)
2852                         continue;
2853
2854                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
2855                         fb = state->base.fb;
2856                         drm_framebuffer_get(fb);
2857                         goto valid_fb;
2858                 }
2859         }
2860
2861         /*
2862          * We've failed to reconstruct the BIOS FB.  Current display state
2863          * indicates that the primary plane is visible, but has a NULL FB,
2864          * which will lead to problems later if we don't fix it up.  The
2865          * simplest solution is to just disable the primary plane now and
2866          * pretend the BIOS never had it enabled.
2867          */
2868         intel_plane_disable_noatomic(intel_crtc, intel_plane);
2869
2870         return;
2871
2872 valid_fb:
2873         intel_fill_fb_ggtt_view(&intel_state->view, fb,
2874                                 intel_state->base.rotation);
2875         intel_state->color_plane[0].stride =
2876                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
2877
2878         mutex_lock(&dev->struct_mutex);
2879         intel_state->vma =
2880                 intel_pin_and_fence_fb_obj(fb,
2881                                            &intel_state->view,
2882                                            intel_plane_uses_fence(intel_state),
2883                                            &intel_state->flags);
2884         mutex_unlock(&dev->struct_mutex);
2885         if (IS_ERR(intel_state->vma)) {
2886                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
2887                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
2888
2889                 intel_state->vma = NULL;
2890                 drm_framebuffer_put(fb);
2891                 return;
2892         }
2893
2894         obj = intel_fb_obj(fb);
2895         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
2896
2897         plane_state->src_x = 0;
2898         plane_state->src_y = 0;
2899         plane_state->src_w = fb->width << 16;
2900         plane_state->src_h = fb->height << 16;
2901
2902         plane_state->crtc_x = 0;
2903         plane_state->crtc_y = 0;
2904         plane_state->crtc_w = fb->width;
2905         plane_state->crtc_h = fb->height;
2906
2907         intel_state->base.src = drm_plane_state_src(plane_state);
2908         intel_state->base.dst = drm_plane_state_dest(plane_state);
2909
2910         if (i915_gem_object_is_tiled(obj))
2911                 dev_priv->preserve_bios_swizzle = true;
2912
2913         plane_state->fb = fb;
2914         plane_state->crtc = &intel_crtc->base;
2915
2916         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
2917                   &obj->frontbuffer_bits);
2918 }
2919
2920 static int skl_max_plane_width(const struct drm_framebuffer *fb,
2921                                int color_plane,
2922                                unsigned int rotation)
2923 {
2924         int cpp = fb->format->cpp[color_plane];
2925
2926         switch (fb->modifier) {
2927         case DRM_FORMAT_MOD_LINEAR:
2928         case I915_FORMAT_MOD_X_TILED:
2929                 switch (cpp) {
2930                 case 8:
2931                         return 4096;
2932                 case 4:
2933                 case 2:
2934                 case 1:
2935                         return 8192;
2936                 default:
2937                         MISSING_CASE(cpp);
2938                         break;
2939                 }
2940                 break;
2941         case I915_FORMAT_MOD_Y_TILED_CCS:
2942         case I915_FORMAT_MOD_Yf_TILED_CCS:
2943                 /* FIXME AUX plane? */
2944         case I915_FORMAT_MOD_Y_TILED:
2945         case I915_FORMAT_MOD_Yf_TILED:
2946                 switch (cpp) {
2947                 case 8:
2948                         return 2048;
2949                 case 4:
2950                         return 4096;
2951                 case 2:
2952                 case 1:
2953                         return 8192;
2954                 default:
2955                         MISSING_CASE(cpp);
2956                         break;
2957                 }
2958                 break;
2959         default:
2960                 MISSING_CASE(fb->modifier);
2961         }
2962
2963         return 2048;
2964 }
2965
2966 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
2967                                            int main_x, int main_y, u32 main_offset)
2968 {
2969         const struct drm_framebuffer *fb = plane_state->base.fb;
2970         int hsub = fb->format->hsub;
2971         int vsub = fb->format->vsub;
2972         int aux_x = plane_state->color_plane[1].x;
2973         int aux_y = plane_state->color_plane[1].y;
2974         u32 aux_offset = plane_state->color_plane[1].offset;
2975         u32 alignment = intel_surf_alignment(fb, 1);
2976
2977         while (aux_offset >= main_offset && aux_y <= main_y) {
2978                 int x, y;
2979
2980                 if (aux_x == main_x && aux_y == main_y)
2981                         break;
2982
2983                 if (aux_offset == 0)
2984                         break;
2985
2986                 x = aux_x / hsub;
2987                 y = aux_y / vsub;
2988                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
2989                                                                aux_offset, aux_offset - alignment);
2990                 aux_x = x * hsub + aux_x % hsub;
2991                 aux_y = y * vsub + aux_y % vsub;
2992         }
2993
2994         if (aux_x != main_x || aux_y != main_y)
2995                 return false;
2996
2997         plane_state->color_plane[1].offset = aux_offset;
2998         plane_state->color_plane[1].x = aux_x;
2999         plane_state->color_plane[1].y = aux_y;
3000
3001         return true;
3002 }
3003
3004 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3005 {
3006         const struct drm_framebuffer *fb = plane_state->base.fb;
3007         unsigned int rotation = plane_state->base.rotation;
3008         int x = plane_state->base.src.x1 >> 16;
3009         int y = plane_state->base.src.y1 >> 16;
3010         int w = drm_rect_width(&plane_state->base.src) >> 16;
3011         int h = drm_rect_height(&plane_state->base.src) >> 16;
3012         int max_width = skl_max_plane_width(fb, 0, rotation);
3013         int max_height = 4096;
3014         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3015
3016         if (w > max_width || h > max_height) {
3017                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3018                               w, h, max_width, max_height);
3019                 return -EINVAL;
3020         }
3021
3022         intel_add_fb_offsets(&x, &y, plane_state, 0);
3023         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3024         alignment = intel_surf_alignment(fb, 0);
3025
3026         /*
3027          * AUX surface offset is specified as the distance from the
3028          * main surface offset, and it must be non-negative. Make
3029          * sure that is what we will get.
3030          */
3031         if (offset > aux_offset)
3032                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3033                                                            offset, aux_offset & ~(alignment - 1));
3034
3035         /*
3036          * When using an X-tiled surface, the plane blows up
3037          * if the x offset + width exceed the stride.
3038          *
3039          * TODO: linear and Y-tiled seem fine, Yf untested,
3040          */
3041         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3042                 int cpp = fb->format->cpp[0];
3043
3044                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3045                         if (offset == 0) {
3046                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3047                                 return -EINVAL;
3048                         }
3049
3050                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3051                                                                    offset, offset - alignment);
3052                 }
3053         }
3054
3055         /*
3056          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3057          * they match with the main surface x/y offsets.
3058          */
3059         if (is_ccs_modifier(fb->modifier)) {
3060                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3061                         if (offset == 0)
3062                                 break;
3063
3064                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3065                                                                    offset, offset - alignment);
3066                 }
3067
3068                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3069                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3070                         return -EINVAL;
3071                 }
3072         }
3073
3074         plane_state->color_plane[0].offset = offset;
3075         plane_state->color_plane[0].x = x;
3076         plane_state->color_plane[0].y = y;
3077
3078         return 0;
3079 }
3080
3081 static int
3082 skl_check_nv12_surface(struct intel_plane_state *plane_state)
3083 {
3084         /* Display WA #1106 */
3085         if (plane_state->base.rotation !=
3086             (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90) &&
3087             plane_state->base.rotation != DRM_MODE_ROTATE_270)
3088                 return 0;
3089
3090         /*
3091          * src coordinates are rotated here.
3092          * We check height but report it as width
3093          */
3094         if (((drm_rect_height(&plane_state->base.src) >> 16) % 4) != 0) {
3095                 DRM_DEBUG_KMS("src width must be multiple "
3096                               "of 4 for rotated NV12\n");
3097                 return -EINVAL;
3098         }
3099
3100         return 0;
3101 }
3102
3103 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3104 {
3105         const struct drm_framebuffer *fb = plane_state->base.fb;
3106         unsigned int rotation = plane_state->base.rotation;
3107         int max_width = skl_max_plane_width(fb, 1, rotation);
3108         int max_height = 4096;
3109         int x = plane_state->base.src.x1 >> 17;
3110         int y = plane_state->base.src.y1 >> 17;
3111         int w = drm_rect_width(&plane_state->base.src) >> 17;
3112         int h = drm_rect_height(&plane_state->base.src) >> 17;
3113         u32 offset;
3114
3115         intel_add_fb_offsets(&x, &y, plane_state, 1);
3116         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3117
3118         /* FIXME not quite sure how/if these apply to the chroma plane */
3119         if (w > max_width || h > max_height) {
3120                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3121                               w, h, max_width, max_height);
3122                 return -EINVAL;
3123         }
3124
3125         plane_state->color_plane[1].offset = offset;
3126         plane_state->color_plane[1].x = x;
3127         plane_state->color_plane[1].y = y;
3128
3129         return 0;
3130 }
3131
3132 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3133 {
3134         const struct drm_framebuffer *fb = plane_state->base.fb;
3135         int src_x = plane_state->base.src.x1 >> 16;
3136         int src_y = plane_state->base.src.y1 >> 16;
3137         int hsub = fb->format->hsub;
3138         int vsub = fb->format->vsub;
3139         int x = src_x / hsub;
3140         int y = src_y / vsub;
3141         u32 offset;
3142
3143         intel_add_fb_offsets(&x, &y, plane_state, 1);
3144         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3145
3146         plane_state->color_plane[1].offset = offset;
3147         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3148         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3149
3150         return 0;
3151 }
3152
3153 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3154 {
3155         const struct drm_framebuffer *fb = plane_state->base.fb;
3156         unsigned int rotation = plane_state->base.rotation;
3157         int ret;
3158
3159         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3160         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3161         plane_state->color_plane[1].stride = intel_fb_pitch(fb, 1, rotation);
3162
3163         ret = intel_plane_check_stride(plane_state);
3164         if (ret)
3165                 return ret;
3166
3167         /* HW only has 8 bits pixel precision, disable plane if invisible */
3168         if (!(plane_state->base.alpha >> 8))
3169                 plane_state->base.visible = false;
3170
3171         if (!plane_state->base.visible)
3172                 return 0;
3173
3174         /* Rotate src coordinates to match rotated GTT view */
3175         if (drm_rotation_90_or_270(rotation))
3176                 drm_rect_rotate(&plane_state->base.src,
3177                                 fb->width << 16, fb->height << 16,
3178                                 DRM_MODE_ROTATE_270);
3179
3180         /*
3181          * Handle the AUX surface first since
3182          * the main surface setup depends on it.
3183          */
3184         if (fb->format->format == DRM_FORMAT_NV12) {
3185                 ret = skl_check_nv12_surface(plane_state);
3186                 if (ret)
3187                         return ret;
3188                 ret = skl_check_nv12_aux_surface(plane_state);
3189                 if (ret)
3190                         return ret;
3191         } else if (is_ccs_modifier(fb->modifier)) {
3192                 ret = skl_check_ccs_aux_surface(plane_state);
3193                 if (ret)
3194                         return ret;
3195         } else {
3196                 plane_state->color_plane[1].offset = ~0xfff;
3197                 plane_state->color_plane[1].x = 0;
3198                 plane_state->color_plane[1].y = 0;
3199         }
3200
3201         ret = skl_check_main_surface(plane_state);
3202         if (ret)
3203                 return ret;
3204
3205         return 0;
3206 }
3207
3208 unsigned int
3209 i9xx_plane_max_stride(struct intel_plane *plane,
3210                       u32 pixel_format, u64 modifier,
3211                       unsigned int rotation)
3212 {
3213         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3214
3215         if (!HAS_GMCH_DISPLAY(dev_priv)) {
3216                 return 32*1024;
3217         } else if (INTEL_GEN(dev_priv) >= 4) {
3218                 if (modifier == I915_FORMAT_MOD_X_TILED)
3219                         return 16*1024;
3220                 else
3221                         return 32*1024;
3222         } else if (INTEL_GEN(dev_priv) >= 3) {
3223                 if (modifier == I915_FORMAT_MOD_X_TILED)
3224                         return 8*1024;
3225                 else
3226                         return 16*1024;
3227         } else {
3228                 if (plane->i9xx_plane == PLANE_C)
3229                         return 4*1024;
3230                 else
3231                         return 8*1024;
3232         }
3233 }
3234
3235 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3236                           const struct intel_plane_state *plane_state)
3237 {
3238         struct drm_i915_private *dev_priv =
3239                 to_i915(plane_state->base.plane->dev);
3240         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3241         const struct drm_framebuffer *fb = plane_state->base.fb;
3242         unsigned int rotation = plane_state->base.rotation;
3243         u32 dspcntr;
3244
3245         dspcntr = DISPLAY_PLANE_ENABLE | DISPPLANE_GAMMA_ENABLE;
3246
3247         if (IS_G4X(dev_priv) || IS_GEN5(dev_priv) ||
3248             IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
3249                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3250
3251         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3252                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3253
3254         if (INTEL_GEN(dev_priv) < 5)
3255                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3256
3257         switch (fb->format->format) {
3258         case DRM_FORMAT_C8:
3259                 dspcntr |= DISPPLANE_8BPP;
3260                 break;
3261         case DRM_FORMAT_XRGB1555:
3262                 dspcntr |= DISPPLANE_BGRX555;
3263                 break;
3264         case DRM_FORMAT_RGB565:
3265                 dspcntr |= DISPPLANE_BGRX565;
3266                 break;
3267         case DRM_FORMAT_XRGB8888:
3268                 dspcntr |= DISPPLANE_BGRX888;
3269                 break;
3270         case DRM_FORMAT_XBGR8888:
3271                 dspcntr |= DISPPLANE_RGBX888;
3272                 break;
3273         case DRM_FORMAT_XRGB2101010:
3274                 dspcntr |= DISPPLANE_BGRX101010;
3275                 break;
3276         case DRM_FORMAT_XBGR2101010:
3277                 dspcntr |= DISPPLANE_RGBX101010;
3278                 break;
3279         default:
3280                 MISSING_CASE(fb->format->format);
3281                 return 0;
3282         }
3283
3284         if (INTEL_GEN(dev_priv) >= 4 &&
3285             fb->modifier == I915_FORMAT_MOD_X_TILED)
3286                 dspcntr |= DISPPLANE_TILED;
3287
3288         if (rotation & DRM_MODE_ROTATE_180)
3289                 dspcntr |= DISPPLANE_ROTATE_180;
3290
3291         if (rotation & DRM_MODE_REFLECT_X)
3292                 dspcntr |= DISPPLANE_MIRROR;
3293
3294         return dspcntr;
3295 }
3296
3297 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3298 {
3299         struct drm_i915_private *dev_priv =
3300                 to_i915(plane_state->base.plane->dev);
3301         const struct drm_framebuffer *fb = plane_state->base.fb;
3302         unsigned int rotation = plane_state->base.rotation;
3303         int src_x = plane_state->base.src.x1 >> 16;
3304         int src_y = plane_state->base.src.y1 >> 16;
3305         u32 offset;
3306         int ret;
3307
3308         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3309         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3310
3311         ret = intel_plane_check_stride(plane_state);
3312         if (ret)
3313                 return ret;
3314
3315         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3316
3317         if (INTEL_GEN(dev_priv) >= 4)
3318                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3319                                                             plane_state, 0);
3320         else
3321                 offset = 0;
3322
3323         /* HSW/BDW do this automagically in hardware */
3324         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3325                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3326                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3327
3328                 if (rotation & DRM_MODE_ROTATE_180) {
3329                         src_x += src_w - 1;
3330                         src_y += src_h - 1;
3331                 } else if (rotation & DRM_MODE_REFLECT_X) {
3332                         src_x += src_w - 1;
3333                 }
3334         }
3335
3336         plane_state->color_plane[0].offset = offset;
3337         plane_state->color_plane[0].x = src_x;
3338         plane_state->color_plane[0].y = src_y;
3339
3340         return 0;
3341 }
3342
3343 static int
3344 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3345                  struct intel_plane_state *plane_state)
3346 {
3347         int ret;
3348
3349         ret = chv_plane_check_rotation(plane_state);
3350         if (ret)
3351                 return ret;
3352
3353         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3354                                                   &crtc_state->base,
3355                                                   DRM_PLANE_HELPER_NO_SCALING,
3356                                                   DRM_PLANE_HELPER_NO_SCALING,
3357                                                   false, true);
3358         if (ret)
3359                 return ret;
3360
3361         if (!plane_state->base.visible)
3362                 return 0;
3363
3364         ret = intel_plane_check_src_coordinates(plane_state);
3365         if (ret)
3366                 return ret;
3367
3368         ret = i9xx_check_plane_surface(plane_state);
3369         if (ret)
3370                 return ret;
3371
3372         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3373
3374         return 0;
3375 }
3376
3377 static void i9xx_update_plane(struct intel_plane *plane,
3378                               const struct intel_crtc_state *crtc_state,
3379                               const struct intel_plane_state *plane_state)
3380 {
3381         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3382         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3383         u32 linear_offset;
3384         u32 dspcntr = plane_state->ctl;
3385         i915_reg_t reg = DSPCNTR(i9xx_plane);
3386         int x = plane_state->color_plane[0].x;
3387         int y = plane_state->color_plane[0].y;
3388         unsigned long irqflags;
3389         u32 dspaddr_offset;
3390
3391         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3392
3393         if (INTEL_GEN(dev_priv) >= 4)
3394                 dspaddr_offset = plane_state->color_plane[0].offset;
3395         else
3396                 dspaddr_offset = linear_offset;
3397
3398         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3399
3400         if (INTEL_GEN(dev_priv) < 4) {
3401                 /* pipesrc and dspsize control the size that is scaled from,
3402                  * which should always be the user's requested size.
3403                  */
3404                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3405                               ((crtc_state->pipe_src_h - 1) << 16) |
3406                               (crtc_state->pipe_src_w - 1));
3407                 I915_WRITE_FW(DSPPOS(i9xx_plane), 0);
3408         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3409                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3410                               ((crtc_state->pipe_src_h - 1) << 16) |
3411                               (crtc_state->pipe_src_w - 1));
3412                 I915_WRITE_FW(PRIMPOS(i9xx_plane), 0);
3413                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3414         }
3415
3416         I915_WRITE_FW(reg, dspcntr);
3417
3418         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3419         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3420                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3421                               intel_plane_ggtt_offset(plane_state) +
3422                               dspaddr_offset);
3423                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3424         } else if (INTEL_GEN(dev_priv) >= 4) {
3425                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3426                               intel_plane_ggtt_offset(plane_state) +
3427                               dspaddr_offset);
3428                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3429                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3430         } else {
3431                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3432                               intel_plane_ggtt_offset(plane_state) +
3433                               dspaddr_offset);
3434         }
3435         POSTING_READ_FW(reg);
3436
3437         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3438 }
3439
3440 static void i9xx_disable_plane(struct intel_plane *plane,
3441                                struct intel_crtc *crtc)
3442 {
3443         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3444         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3445         unsigned long irqflags;
3446
3447         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3448
3449         I915_WRITE_FW(DSPCNTR(i9xx_plane), 0);
3450         if (INTEL_GEN(dev_priv) >= 4)
3451                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3452         else
3453                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3454         POSTING_READ_FW(DSPCNTR(i9xx_plane));
3455
3456         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3457 }
3458
3459 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3460                                     enum pipe *pipe)
3461 {
3462         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3463         enum intel_display_power_domain power_domain;
3464         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3465         bool ret;
3466         u32 val;
3467
3468         /*
3469          * Not 100% correct for planes that can move between pipes,
3470          * but that's only the case for gen2-4 which don't have any
3471          * display power wells.
3472          */
3473         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3474         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3475                 return false;
3476
3477         val = I915_READ(DSPCNTR(i9xx_plane));
3478
3479         ret = val & DISPLAY_PLANE_ENABLE;
3480
3481         if (INTEL_GEN(dev_priv) >= 5)
3482                 *pipe = plane->pipe;
3483         else
3484                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3485                         DISPPLANE_SEL_PIPE_SHIFT;
3486
3487         intel_display_power_put(dev_priv, power_domain);
3488
3489         return ret;
3490 }
3491
3492 static u32
3493 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
3494 {
3495         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3496                 return 64;
3497         else
3498                 return intel_tile_width_bytes(fb, color_plane);
3499 }
3500
3501 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3502 {
3503         struct drm_device *dev = intel_crtc->base.dev;
3504         struct drm_i915_private *dev_priv = to_i915(dev);
3505
3506         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3507         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3508         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3509 }
3510
3511 /*
3512  * This function detaches (aka. unbinds) unused scalers in hardware
3513  */
3514 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
3515 {
3516         struct intel_crtc_scaler_state *scaler_state;
3517         int i;
3518
3519         scaler_state = &intel_crtc->config->scaler_state;
3520
3521         /* loop through and disable scalers that aren't in use */
3522         for (i = 0; i < intel_crtc->num_scalers; i++) {
3523                 if (!scaler_state->scalers[i].in_use)
3524                         skl_detach_scaler(intel_crtc, i);
3525         }
3526 }
3527
3528 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3529                      int color_plane)
3530 {
3531         const struct drm_framebuffer *fb = plane_state->base.fb;
3532         unsigned int rotation = plane_state->base.rotation;
3533         u32 stride = plane_state->color_plane[color_plane].stride;
3534
3535         if (color_plane >= fb->format->num_planes)
3536                 return 0;
3537
3538         /*
3539          * The stride is either expressed as a multiple of 64 bytes chunks for
3540          * linear buffers or in number of tiles for tiled buffers.
3541          */
3542         if (drm_rotation_90_or_270(rotation))
3543                 stride /= intel_tile_height(fb, color_plane);
3544         else
3545                 stride /= intel_fb_stride_alignment(fb, color_plane);
3546
3547         return stride;
3548 }
3549
3550 static u32 skl_plane_ctl_format(uint32_t pixel_format)
3551 {
3552         switch (pixel_format) {
3553         case DRM_FORMAT_C8:
3554                 return PLANE_CTL_FORMAT_INDEXED;
3555         case DRM_FORMAT_RGB565:
3556                 return PLANE_CTL_FORMAT_RGB_565;
3557         case DRM_FORMAT_XBGR8888:
3558         case DRM_FORMAT_ABGR8888:
3559                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3560         case DRM_FORMAT_XRGB8888:
3561         case DRM_FORMAT_ARGB8888:
3562                 return PLANE_CTL_FORMAT_XRGB_8888;
3563         case DRM_FORMAT_XRGB2101010:
3564                 return PLANE_CTL_FORMAT_XRGB_2101010;
3565         case DRM_FORMAT_XBGR2101010:
3566                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3567         case DRM_FORMAT_YUYV:
3568                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3569         case DRM_FORMAT_YVYU:
3570                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3571         case DRM_FORMAT_UYVY:
3572                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3573         case DRM_FORMAT_VYUY:
3574                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3575         case DRM_FORMAT_NV12:
3576                 return PLANE_CTL_FORMAT_NV12;
3577         default:
3578                 MISSING_CASE(pixel_format);
3579         }
3580
3581         return 0;
3582 }
3583
3584 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
3585 {
3586         if (!plane_state->base.fb->format->has_alpha)
3587                 return PLANE_CTL_ALPHA_DISABLE;
3588
3589         switch (plane_state->base.pixel_blend_mode) {
3590         case DRM_MODE_BLEND_PIXEL_NONE:
3591                 return PLANE_CTL_ALPHA_DISABLE;
3592         case DRM_MODE_BLEND_PREMULTI:
3593                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3594         case DRM_MODE_BLEND_COVERAGE:
3595                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
3596         default:
3597                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3598                 return PLANE_CTL_ALPHA_DISABLE;
3599         }
3600 }
3601
3602 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
3603 {
3604         if (!plane_state->base.fb->format->has_alpha)
3605                 return PLANE_COLOR_ALPHA_DISABLE;
3606
3607         switch (plane_state->base.pixel_blend_mode) {
3608         case DRM_MODE_BLEND_PIXEL_NONE:
3609                 return PLANE_COLOR_ALPHA_DISABLE;
3610         case DRM_MODE_BLEND_PREMULTI:
3611                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
3612         case DRM_MODE_BLEND_COVERAGE:
3613                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
3614         default:
3615                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3616                 return PLANE_COLOR_ALPHA_DISABLE;
3617         }
3618 }
3619
3620 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3621 {
3622         switch (fb_modifier) {
3623         case DRM_FORMAT_MOD_LINEAR:
3624                 break;
3625         case I915_FORMAT_MOD_X_TILED:
3626                 return PLANE_CTL_TILED_X;
3627         case I915_FORMAT_MOD_Y_TILED:
3628                 return PLANE_CTL_TILED_Y;
3629         case I915_FORMAT_MOD_Y_TILED_CCS:
3630                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3631         case I915_FORMAT_MOD_Yf_TILED:
3632                 return PLANE_CTL_TILED_YF;
3633         case I915_FORMAT_MOD_Yf_TILED_CCS:
3634                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3635         default:
3636                 MISSING_CASE(fb_modifier);
3637         }
3638
3639         return 0;
3640 }
3641
3642 static u32 skl_plane_ctl_rotate(unsigned int rotate)
3643 {
3644         switch (rotate) {
3645         case DRM_MODE_ROTATE_0:
3646                 break;
3647         /*
3648          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
3649          * while i915 HW rotation is clockwise, thats why this swapping.
3650          */
3651         case DRM_MODE_ROTATE_90:
3652                 return PLANE_CTL_ROTATE_270;
3653         case DRM_MODE_ROTATE_180:
3654                 return PLANE_CTL_ROTATE_180;
3655         case DRM_MODE_ROTATE_270:
3656                 return PLANE_CTL_ROTATE_90;
3657         default:
3658                 MISSING_CASE(rotate);
3659         }
3660
3661         return 0;
3662 }
3663
3664 static u32 cnl_plane_ctl_flip(unsigned int reflect)
3665 {
3666         switch (reflect) {
3667         case 0:
3668                 break;
3669         case DRM_MODE_REFLECT_X:
3670                 return PLANE_CTL_FLIP_HORIZONTAL;
3671         case DRM_MODE_REFLECT_Y:
3672         default:
3673                 MISSING_CASE(reflect);
3674         }
3675
3676         return 0;
3677 }
3678
3679 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
3680                   const struct intel_plane_state *plane_state)
3681 {
3682         struct drm_i915_private *dev_priv =
3683                 to_i915(plane_state->base.plane->dev);
3684         const struct drm_framebuffer *fb = plane_state->base.fb;
3685         unsigned int rotation = plane_state->base.rotation;
3686         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
3687         u32 plane_ctl;
3688
3689         plane_ctl = PLANE_CTL_ENABLE;
3690
3691         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
3692                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
3693                 plane_ctl |=
3694                         PLANE_CTL_PIPE_GAMMA_ENABLE |
3695                         PLANE_CTL_PIPE_CSC_ENABLE |
3696                         PLANE_CTL_PLANE_GAMMA_DISABLE;
3697
3698                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3699                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
3700
3701                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3702                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
3703         }
3704
3705         plane_ctl |= skl_plane_ctl_format(fb->format->format);
3706         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
3707         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
3708
3709         if (INTEL_GEN(dev_priv) >= 10)
3710                 plane_ctl |= cnl_plane_ctl_flip(rotation &
3711                                                 DRM_MODE_REFLECT_MASK);
3712
3713         if (key->flags & I915_SET_COLORKEY_DESTINATION)
3714                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
3715         else if (key->flags & I915_SET_COLORKEY_SOURCE)
3716                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
3717
3718         return plane_ctl;
3719 }
3720
3721 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
3722                         const struct intel_plane_state *plane_state)
3723 {
3724         struct drm_i915_private *dev_priv =
3725                 to_i915(plane_state->base.plane->dev);
3726         const struct drm_framebuffer *fb = plane_state->base.fb;
3727         u32 plane_color_ctl = 0;
3728
3729         if (INTEL_GEN(dev_priv) < 11) {
3730                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
3731                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
3732         }
3733         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
3734         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
3735
3736         if (fb->format->is_yuv) {
3737                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3738                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
3739                 else
3740                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
3741
3742                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3743                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3744         }
3745
3746         return plane_color_ctl;
3747 }
3748
3749 static int
3750 __intel_display_resume(struct drm_device *dev,
3751                        struct drm_atomic_state *state,
3752                        struct drm_modeset_acquire_ctx *ctx)
3753 {
3754         struct drm_crtc_state *crtc_state;
3755         struct drm_crtc *crtc;
3756         int i, ret;
3757
3758         intel_modeset_setup_hw_state(dev, ctx);
3759         i915_redisable_vga(to_i915(dev));
3760
3761         if (!state)
3762                 return 0;
3763
3764         /*
3765          * We've duplicated the state, pointers to the old state are invalid.
3766          *
3767          * Don't attempt to use the old state until we commit the duplicated state.
3768          */
3769         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3770                 /*
3771                  * Force recalculation even if we restore
3772                  * current state. With fast modeset this may not result
3773                  * in a modeset when the state is compatible.
3774                  */
3775                 crtc_state->mode_changed = true;
3776         }
3777
3778         /* ignore any reset values/BIOS leftovers in the WM registers */
3779         if (!HAS_GMCH_DISPLAY(to_i915(dev)))
3780                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
3781
3782         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
3783
3784         WARN_ON(ret == -EDEADLK);
3785         return ret;
3786 }
3787
3788 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
3789 {
3790         return intel_has_gpu_reset(dev_priv) &&
3791                 INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv);
3792 }
3793
3794 void intel_prepare_reset(struct drm_i915_private *dev_priv)
3795 {
3796         struct drm_device *dev = &dev_priv->drm;
3797         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3798         struct drm_atomic_state *state;
3799         int ret;
3800
3801         /* reset doesn't touch the display */
3802         if (!i915_modparams.force_reset_modeset_test &&
3803             !gpu_reset_clobbers_display(dev_priv))
3804                 return;
3805
3806         /* We have a modeset vs reset deadlock, defensively unbreak it. */
3807         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3808         wake_up_all(&dev_priv->gpu_error.wait_queue);
3809
3810         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
3811                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
3812                 i915_gem_set_wedged(dev_priv);
3813         }
3814
3815         /*
3816          * Need mode_config.mutex so that we don't
3817          * trample ongoing ->detect() and whatnot.
3818          */
3819         mutex_lock(&dev->mode_config.mutex);
3820         drm_modeset_acquire_init(ctx, 0);
3821         while (1) {
3822                 ret = drm_modeset_lock_all_ctx(dev, ctx);
3823                 if (ret != -EDEADLK)
3824                         break;
3825
3826                 drm_modeset_backoff(ctx);
3827         }
3828         /*
3829          * Disabling the crtcs gracefully seems nicer. Also the
3830          * g33 docs say we should at least disable all the planes.
3831          */
3832         state = drm_atomic_helper_duplicate_state(dev, ctx);
3833         if (IS_ERR(state)) {
3834                 ret = PTR_ERR(state);
3835                 DRM_ERROR("Duplicating state failed with %i\n", ret);
3836                 return;
3837         }
3838
3839         ret = drm_atomic_helper_disable_all(dev, ctx);
3840         if (ret) {
3841                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
3842                 drm_atomic_state_put(state);
3843                 return;
3844         }
3845
3846         dev_priv->modeset_restore_state = state;
3847         state->acquire_ctx = ctx;
3848 }
3849
3850 void intel_finish_reset(struct drm_i915_private *dev_priv)
3851 {
3852         struct drm_device *dev = &dev_priv->drm;
3853         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3854         struct drm_atomic_state *state;
3855         int ret;
3856
3857         /* reset doesn't touch the display */
3858         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
3859                 return;
3860
3861         state = fetch_and_zero(&dev_priv->modeset_restore_state);
3862         if (!state)
3863                 goto unlock;
3864
3865         /* reset doesn't touch the display */
3866         if (!gpu_reset_clobbers_display(dev_priv)) {
3867                 /* for testing only restore the display */
3868                 ret = __intel_display_resume(dev, state, ctx);
3869                 if (ret)
3870                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3871         } else {
3872                 /*
3873                  * The display has been reset as well,
3874                  * so need a full re-initialization.
3875                  */
3876                 intel_runtime_pm_disable_interrupts(dev_priv);
3877                 intel_runtime_pm_enable_interrupts(dev_priv);
3878
3879                 intel_pps_unlock_regs_wa(dev_priv);
3880                 intel_modeset_init_hw(dev);
3881                 intel_init_clock_gating(dev_priv);
3882
3883                 spin_lock_irq(&dev_priv->irq_lock);
3884                 if (dev_priv->display.hpd_irq_setup)
3885                         dev_priv->display.hpd_irq_setup(dev_priv);
3886                 spin_unlock_irq(&dev_priv->irq_lock);
3887
3888                 ret = __intel_display_resume(dev, state, ctx);
3889                 if (ret)
3890                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3891
3892                 intel_hpd_init(dev_priv);
3893         }
3894
3895         drm_atomic_state_put(state);
3896 unlock:
3897         drm_modeset_drop_locks(ctx);
3898         drm_modeset_acquire_fini(ctx);
3899         mutex_unlock(&dev->mode_config.mutex);
3900
3901         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3902 }
3903
3904 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
3905                                      const struct intel_crtc_state *new_crtc_state)
3906 {
3907         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
3908         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3909
3910         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3911         crtc->base.mode = new_crtc_state->base.mode;
3912
3913         /*
3914          * Update pipe size and adjust fitter if needed: the reason for this is
3915          * that in compute_mode_changes we check the native mode (not the pfit
3916          * mode) to see if we can flip rather than do a full mode set. In the
3917          * fastboot case, we'll flip, but if we don't update the pipesrc and
3918          * pfit state, we'll end up with a big fb scanned out into the wrong
3919          * sized surface.
3920          */
3921
3922         I915_WRITE(PIPESRC(crtc->pipe),
3923                    ((new_crtc_state->pipe_src_w - 1) << 16) |
3924                    (new_crtc_state->pipe_src_h - 1));
3925
3926         /* on skylake this is done by detaching scalers */
3927         if (INTEL_GEN(dev_priv) >= 9) {
3928                 skl_detach_scalers(crtc);
3929
3930                 if (new_crtc_state->pch_pfit.enabled)
3931                         skylake_pfit_enable(new_crtc_state);
3932         } else if (HAS_PCH_SPLIT(dev_priv)) {
3933                 if (new_crtc_state->pch_pfit.enabled)
3934                         ironlake_pfit_enable(new_crtc_state);
3935                 else if (old_crtc_state->pch_pfit.enabled)
3936                         ironlake_pfit_disable(old_crtc_state);
3937         }
3938 }
3939
3940 static void intel_fdi_normal_train(struct intel_crtc *crtc)
3941 {
3942         struct drm_device *dev = crtc->base.dev;
3943         struct drm_i915_private *dev_priv = to_i915(dev);
3944         int pipe = crtc->pipe;
3945         i915_reg_t reg;
3946         u32 temp;
3947
3948         /* enable normal train */
3949         reg = FDI_TX_CTL(pipe);
3950         temp = I915_READ(reg);
3951         if (IS_IVYBRIDGE(dev_priv)) {
3952                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3953                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3954         } else {
3955                 temp &= ~FDI_LINK_TRAIN_NONE;
3956                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3957         }
3958         I915_WRITE(reg, temp);
3959
3960         reg = FDI_RX_CTL(pipe);
3961         temp = I915_READ(reg);
3962         if (HAS_PCH_CPT(dev_priv)) {
3963                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3964                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3965         } else {
3966                 temp &= ~FDI_LINK_TRAIN_NONE;
3967                 temp |= FDI_LINK_TRAIN_NONE;
3968         }
3969         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3970
3971         /* wait one idle pattern time */
3972         POSTING_READ(reg);
3973         udelay(1000);
3974
3975         /* IVB wants error correction enabled */
3976         if (IS_IVYBRIDGE(dev_priv))
3977                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3978                            FDI_FE_ERRC_ENABLE);
3979 }
3980
3981 /* The FDI link training functions for ILK/Ibexpeak. */
3982 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
3983                                     const struct intel_crtc_state *crtc_state)
3984 {
3985         struct drm_device *dev = crtc->base.dev;
3986         struct drm_i915_private *dev_priv = to_i915(dev);
3987         int pipe = crtc->pipe;
3988         i915_reg_t reg;
3989         u32 temp, tries;
3990
3991         /* FDI needs bits from pipe first */
3992         assert_pipe_enabled(dev_priv, pipe);
3993
3994         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3995            for train result */
3996         reg = FDI_RX_IMR(pipe);
3997         temp = I915_READ(reg);
3998         temp &= ~FDI_RX_SYMBOL_LOCK;
3999         temp &= ~FDI_RX_BIT_LOCK;
4000         I915_WRITE(reg, temp);
4001         I915_READ(reg);
4002         udelay(150);
4003
4004         /* enable CPU FDI TX and PCH FDI RX */
4005         reg = FDI_TX_CTL(pipe);
4006         temp = I915_READ(reg);
4007         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4008         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4009         temp &= ~FDI_LINK_TRAIN_NONE;
4010         temp |= FDI_LINK_TRAIN_PATTERN_1;
4011         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4012
4013         reg = FDI_RX_CTL(pipe);
4014         temp = I915_READ(reg);
4015         temp &= ~FDI_LINK_TRAIN_NONE;
4016         temp |= FDI_LINK_TRAIN_PATTERN_1;
4017         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4018
4019         POSTING_READ(reg);
4020         udelay(150);
4021
4022         /* Ironlake workaround, enable clock pointer after FDI enable*/
4023         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4024         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4025                    FDI_RX_PHASE_SYNC_POINTER_EN);
4026
4027         reg = FDI_RX_IIR(pipe);
4028         for (tries = 0; tries < 5; tries++) {
4029                 temp = I915_READ(reg);
4030                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4031
4032                 if ((temp & FDI_RX_BIT_LOCK)) {
4033                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4034                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4035                         break;
4036                 }
4037         }
4038         if (tries == 5)
4039                 DRM_ERROR("FDI train 1 fail!\n");
4040
4041         /* Train 2 */
4042         reg = FDI_TX_CTL(pipe);
4043         temp = I915_READ(reg);
4044         temp &= ~FDI_LINK_TRAIN_NONE;
4045         temp |= FDI_LINK_TRAIN_PATTERN_2;
4046         I915_WRITE(reg, temp);
4047
4048         reg = FDI_RX_CTL(pipe);
4049         temp = I915_READ(reg);
4050         temp &= ~FDI_LINK_TRAIN_NONE;
4051         temp |= FDI_LINK_TRAIN_PATTERN_2;
4052         I915_WRITE(reg, temp);
4053
4054         POSTING_READ(reg);
4055         udelay(150);
4056
4057         reg = FDI_RX_IIR(pipe);
4058         for (tries = 0; tries < 5; tries++) {
4059                 temp = I915_READ(reg);
4060                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4061
4062                 if (temp & FDI_RX_SYMBOL_LOCK) {
4063                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4064                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4065                         break;
4066                 }
4067         }
4068         if (tries == 5)
4069                 DRM_ERROR("FDI train 2 fail!\n");
4070
4071         DRM_DEBUG_KMS("FDI train done\n");
4072
4073 }
4074
4075 static const int snb_b_fdi_train_param[] = {
4076         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4077         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4078         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4079         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4080 };
4081
4082 /* The FDI link training functions for SNB/Cougarpoint. */
4083 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4084                                 const struct intel_crtc_state *crtc_state)
4085 {
4086         struct drm_device *dev = crtc->base.dev;
4087         struct drm_i915_private *dev_priv = to_i915(dev);
4088         int pipe = crtc->pipe;
4089         i915_reg_t reg;
4090         u32 temp, i, retry;
4091
4092         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4093            for train result */
4094         reg = FDI_RX_IMR(pipe);
4095         temp = I915_READ(reg);
4096         temp &= ~FDI_RX_SYMBOL_LOCK;
4097         temp &= ~FDI_RX_BIT_LOCK;
4098         I915_WRITE(reg, temp);
4099
4100         POSTING_READ(reg);
4101         udelay(150);
4102
4103         /* enable CPU FDI TX and PCH FDI RX */
4104         reg = FDI_TX_CTL(pipe);
4105         temp = I915_READ(reg);
4106         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4107         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4108         temp &= ~FDI_LINK_TRAIN_NONE;
4109         temp |= FDI_LINK_TRAIN_PATTERN_1;
4110         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4111         /* SNB-B */
4112         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4113         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4114
4115         I915_WRITE(FDI_RX_MISC(pipe),
4116                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4117
4118         reg = FDI_RX_CTL(pipe);
4119         temp = I915_READ(reg);
4120         if (HAS_PCH_CPT(dev_priv)) {
4121                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4122                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4123         } else {
4124                 temp &= ~FDI_LINK_TRAIN_NONE;
4125                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4126         }
4127         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4128
4129         POSTING_READ(reg);
4130         udelay(150);
4131
4132         for (i = 0; i < 4; i++) {
4133                 reg = FDI_TX_CTL(pipe);
4134                 temp = I915_READ(reg);
4135                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4136                 temp |= snb_b_fdi_train_param[i];
4137                 I915_WRITE(reg, temp);
4138
4139                 POSTING_READ(reg);
4140                 udelay(500);
4141
4142                 for (retry = 0; retry < 5; retry++) {
4143                         reg = FDI_RX_IIR(pipe);
4144                         temp = I915_READ(reg);
4145                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4146                         if (temp & FDI_RX_BIT_LOCK) {
4147                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4148                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4149                                 break;
4150                         }
4151                         udelay(50);
4152                 }
4153                 if (retry < 5)
4154                         break;
4155         }
4156         if (i == 4)
4157                 DRM_ERROR("FDI train 1 fail!\n");
4158
4159         /* Train 2 */
4160         reg = FDI_TX_CTL(pipe);
4161         temp = I915_READ(reg);
4162         temp &= ~FDI_LINK_TRAIN_NONE;
4163         temp |= FDI_LINK_TRAIN_PATTERN_2;
4164         if (IS_GEN6(dev_priv)) {
4165                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4166                 /* SNB-B */
4167                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4168         }
4169         I915_WRITE(reg, temp);
4170
4171         reg = FDI_RX_CTL(pipe);
4172         temp = I915_READ(reg);
4173         if (HAS_PCH_CPT(dev_priv)) {
4174                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4175                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4176         } else {
4177                 temp &= ~FDI_LINK_TRAIN_NONE;
4178                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4179         }
4180         I915_WRITE(reg, temp);
4181
4182         POSTING_READ(reg);
4183         udelay(150);
4184
4185         for (i = 0; i < 4; i++) {
4186                 reg = FDI_TX_CTL(pipe);
4187                 temp = I915_READ(reg);
4188                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4189                 temp |= snb_b_fdi_train_param[i];
4190                 I915_WRITE(reg, temp);
4191
4192                 POSTING_READ(reg);
4193                 udelay(500);
4194
4195                 for (retry = 0; retry < 5; retry++) {
4196                         reg = FDI_RX_IIR(pipe);
4197                         temp = I915_READ(reg);
4198                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4199                         if (temp & FDI_RX_SYMBOL_LOCK) {
4200                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4201                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4202                                 break;
4203                         }
4204                         udelay(50);
4205                 }
4206                 if (retry < 5)
4207                         break;
4208         }
4209         if (i == 4)
4210                 DRM_ERROR("FDI train 2 fail!\n");
4211
4212         DRM_DEBUG_KMS("FDI train done.\n");
4213 }
4214
4215 /* Manual link training for Ivy Bridge A0 parts */
4216 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4217                                       const struct intel_crtc_state *crtc_state)
4218 {
4219         struct drm_device *dev = crtc->base.dev;
4220         struct drm_i915_private *dev_priv = to_i915(dev);
4221         int pipe = crtc->pipe;
4222         i915_reg_t reg;
4223         u32 temp, i, j;
4224
4225         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4226            for train result */
4227         reg = FDI_RX_IMR(pipe);
4228         temp = I915_READ(reg);
4229         temp &= ~FDI_RX_SYMBOL_LOCK;
4230         temp &= ~FDI_RX_BIT_LOCK;
4231         I915_WRITE(reg, temp);
4232
4233         POSTING_READ(reg);
4234         udelay(150);
4235
4236         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4237                       I915_READ(FDI_RX_IIR(pipe)));
4238
4239         /* Try each vswing and preemphasis setting twice before moving on */
4240         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4241                 /* disable first in case we need to retry */
4242                 reg = FDI_TX_CTL(pipe);
4243                 temp = I915_READ(reg);
4244                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4245                 temp &= ~FDI_TX_ENABLE;
4246                 I915_WRITE(reg, temp);
4247
4248                 reg = FDI_RX_CTL(pipe);
4249                 temp = I915_READ(reg);
4250                 temp &= ~FDI_LINK_TRAIN_AUTO;
4251                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4252                 temp &= ~FDI_RX_ENABLE;
4253                 I915_WRITE(reg, temp);
4254
4255                 /* enable CPU FDI TX and PCH FDI RX */
4256                 reg = FDI_TX_CTL(pipe);
4257                 temp = I915_READ(reg);
4258                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4259                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4260                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4261                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4262                 temp |= snb_b_fdi_train_param[j/2];
4263                 temp |= FDI_COMPOSITE_SYNC;
4264                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4265
4266                 I915_WRITE(FDI_RX_MISC(pipe),
4267                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4268
4269                 reg = FDI_RX_CTL(pipe);
4270                 temp = I915_READ(reg);
4271                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4272                 temp |= FDI_COMPOSITE_SYNC;
4273                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4274
4275                 POSTING_READ(reg);
4276                 udelay(1); /* should be 0.5us */
4277
4278                 for (i = 0; i < 4; i++) {
4279                         reg = FDI_RX_IIR(pipe);
4280                         temp = I915_READ(reg);
4281                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4282
4283                         if (temp & FDI_RX_BIT_LOCK ||
4284                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4285                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4286                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4287                                               i);
4288                                 break;
4289                         }
4290                         udelay(1); /* should be 0.5us */
4291                 }
4292                 if (i == 4) {
4293                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4294                         continue;
4295                 }
4296
4297                 /* Train 2 */
4298                 reg = FDI_TX_CTL(pipe);
4299                 temp = I915_READ(reg);
4300                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4301                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4302                 I915_WRITE(reg, temp);
4303
4304                 reg = FDI_RX_CTL(pipe);
4305                 temp = I915_READ(reg);
4306                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4307                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4308                 I915_WRITE(reg, temp);
4309
4310                 POSTING_READ(reg);
4311                 udelay(2); /* should be 1.5us */
4312
4313                 for (i = 0; i < 4; i++) {
4314                         reg = FDI_RX_IIR(pipe);
4315                         temp = I915_READ(reg);
4316                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4317
4318                         if (temp & FDI_RX_SYMBOL_LOCK ||
4319                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4320                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4321                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4322                                               i);
4323                                 goto train_done;
4324                         }
4325                         udelay(2); /* should be 1.5us */
4326                 }
4327                 if (i == 4)
4328                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4329         }
4330
4331 train_done:
4332         DRM_DEBUG_KMS("FDI train done.\n");
4333 }
4334
4335 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
4336 {
4337         struct drm_device *dev = intel_crtc->base.dev;
4338         struct drm_i915_private *dev_priv = to_i915(dev);
4339         int pipe = intel_crtc->pipe;
4340         i915_reg_t reg;
4341         u32 temp;
4342
4343         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4344         reg = FDI_RX_CTL(pipe);
4345         temp = I915_READ(reg);
4346         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4347         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
4348         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4349         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4350
4351         POSTING_READ(reg);
4352         udelay(200);
4353
4354         /* Switch from Rawclk to PCDclk */
4355         temp = I915_READ(reg);
4356         I915_WRITE(reg, temp | FDI_PCDCLK);
4357
4358         POSTING_READ(reg);
4359         udelay(200);
4360
4361         /* Enable CPU FDI TX PLL, always on for Ironlake */
4362         reg = FDI_TX_CTL(pipe);
4363         temp = I915_READ(reg);
4364         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4365                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4366
4367                 POSTING_READ(reg);
4368                 udelay(100);
4369         }
4370 }
4371
4372 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4373 {
4374         struct drm_device *dev = intel_crtc->base.dev;
4375         struct drm_i915_private *dev_priv = to_i915(dev);
4376         int pipe = intel_crtc->pipe;
4377         i915_reg_t reg;
4378         u32 temp;
4379
4380         /* Switch from PCDclk to Rawclk */
4381         reg = FDI_RX_CTL(pipe);
4382         temp = I915_READ(reg);
4383         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4384
4385         /* Disable CPU FDI TX PLL */
4386         reg = FDI_TX_CTL(pipe);
4387         temp = I915_READ(reg);
4388         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4389
4390         POSTING_READ(reg);
4391         udelay(100);
4392
4393         reg = FDI_RX_CTL(pipe);
4394         temp = I915_READ(reg);
4395         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4396
4397         /* Wait for the clocks to turn off. */
4398         POSTING_READ(reg);
4399         udelay(100);
4400 }
4401
4402 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4403 {
4404         struct drm_device *dev = crtc->dev;
4405         struct drm_i915_private *dev_priv = to_i915(dev);
4406         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4407         int pipe = intel_crtc->pipe;
4408         i915_reg_t reg;
4409         u32 temp;
4410
4411         /* disable CPU FDI tx and PCH FDI rx */
4412         reg = FDI_TX_CTL(pipe);
4413         temp = I915_READ(reg);
4414         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4415         POSTING_READ(reg);
4416
4417         reg = FDI_RX_CTL(pipe);
4418         temp = I915_READ(reg);
4419         temp &= ~(0x7 << 16);
4420         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4421         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4422
4423         POSTING_READ(reg);
4424         udelay(100);
4425
4426         /* Ironlake workaround, disable clock pointer after downing FDI */
4427         if (HAS_PCH_IBX(dev_priv))
4428                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4429
4430         /* still set train pattern 1 */
4431         reg = FDI_TX_CTL(pipe);
4432         temp = I915_READ(reg);
4433         temp &= ~FDI_LINK_TRAIN_NONE;
4434         temp |= FDI_LINK_TRAIN_PATTERN_1;
4435         I915_WRITE(reg, temp);
4436
4437         reg = FDI_RX_CTL(pipe);
4438         temp = I915_READ(reg);
4439         if (HAS_PCH_CPT(dev_priv)) {
4440                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4441                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4442         } else {
4443                 temp &= ~FDI_LINK_TRAIN_NONE;
4444                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4445         }
4446         /* BPC in FDI rx is consistent with that in PIPECONF */
4447         temp &= ~(0x07 << 16);
4448         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4449         I915_WRITE(reg, temp);
4450
4451         POSTING_READ(reg);
4452         udelay(100);
4453 }
4454
4455 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4456 {
4457         struct drm_crtc *crtc;
4458         bool cleanup_done;
4459
4460         drm_for_each_crtc(crtc, &dev_priv->drm) {
4461                 struct drm_crtc_commit *commit;
4462                 spin_lock(&crtc->commit_lock);
4463                 commit = list_first_entry_or_null(&crtc->commit_list,
4464                                                   struct drm_crtc_commit, commit_entry);
4465                 cleanup_done = commit ?
4466                         try_wait_for_completion(&commit->cleanup_done) : true;
4467                 spin_unlock(&crtc->commit_lock);
4468
4469                 if (cleanup_done)
4470                         continue;
4471
4472                 drm_crtc_wait_one_vblank(crtc);
4473
4474                 return true;
4475         }
4476
4477         return false;
4478 }
4479
4480 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4481 {
4482         u32 temp;
4483
4484         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4485
4486         mutex_lock(&dev_priv->sb_lock);
4487
4488         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4489         temp |= SBI_SSCCTL_DISABLE;
4490         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4491
4492         mutex_unlock(&dev_priv->sb_lock);
4493 }
4494
4495 /* Program iCLKIP clock to the desired frequency */
4496 static void lpt_program_iclkip(struct intel_crtc *crtc)
4497 {
4498         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4499         int clock = crtc->config->base.adjusted_mode.crtc_clock;
4500         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4501         u32 temp;
4502
4503         lpt_disable_iclkip(dev_priv);
4504
4505         /* The iCLK virtual clock root frequency is in MHz,
4506          * but the adjusted_mode->crtc_clock in in KHz. To get the
4507          * divisors, it is necessary to divide one by another, so we
4508          * convert the virtual clock precision to KHz here for higher
4509          * precision.
4510          */
4511         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
4512                 u32 iclk_virtual_root_freq = 172800 * 1000;
4513                 u32 iclk_pi_range = 64;
4514                 u32 desired_divisor;
4515
4516                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4517                                                     clock << auxdiv);
4518                 divsel = (desired_divisor / iclk_pi_range) - 2;
4519                 phaseinc = desired_divisor % iclk_pi_range;
4520
4521                 /*
4522                  * Near 20MHz is a corner case which is
4523                  * out of range for the 7-bit divisor
4524                  */
4525                 if (divsel <= 0x7f)
4526                         break;
4527         }
4528
4529         /* This should not happen with any sane values */
4530         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4531                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4532         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4533                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4534
4535         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4536                         clock,
4537                         auxdiv,
4538                         divsel,
4539                         phasedir,
4540                         phaseinc);
4541
4542         mutex_lock(&dev_priv->sb_lock);
4543
4544         /* Program SSCDIVINTPHASE6 */
4545         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4546         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4547         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4548         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4549         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4550         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4551         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4552         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4553
4554         /* Program SSCAUXDIV */
4555         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4556         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4557         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4558         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4559
4560         /* Enable modulator and associated divider */
4561         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4562         temp &= ~SBI_SSCCTL_DISABLE;
4563         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4564
4565         mutex_unlock(&dev_priv->sb_lock);
4566
4567         /* Wait for initialization time */
4568         udelay(24);
4569
4570         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4571 }
4572
4573 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
4574 {
4575         u32 divsel, phaseinc, auxdiv;
4576         u32 iclk_virtual_root_freq = 172800 * 1000;
4577         u32 iclk_pi_range = 64;
4578         u32 desired_divisor;
4579         u32 temp;
4580
4581         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
4582                 return 0;
4583
4584         mutex_lock(&dev_priv->sb_lock);
4585
4586         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4587         if (temp & SBI_SSCCTL_DISABLE) {
4588                 mutex_unlock(&dev_priv->sb_lock);
4589                 return 0;
4590         }
4591
4592         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4593         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
4594                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
4595         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
4596                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
4597
4598         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4599         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
4600                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
4601
4602         mutex_unlock(&dev_priv->sb_lock);
4603
4604         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
4605
4606         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4607                                  desired_divisor << auxdiv);
4608 }
4609
4610 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
4611                                                 enum pipe pch_transcoder)
4612 {
4613         struct drm_device *dev = crtc->base.dev;
4614         struct drm_i915_private *dev_priv = to_i915(dev);
4615         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
4616
4617         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4618                    I915_READ(HTOTAL(cpu_transcoder)));
4619         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4620                    I915_READ(HBLANK(cpu_transcoder)));
4621         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4622                    I915_READ(HSYNC(cpu_transcoder)));
4623
4624         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4625                    I915_READ(VTOTAL(cpu_transcoder)));
4626         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4627                    I915_READ(VBLANK(cpu_transcoder)));
4628         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4629                    I915_READ(VSYNC(cpu_transcoder)));
4630         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4631                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4632 }
4633
4634 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4635 {
4636         struct drm_i915_private *dev_priv = to_i915(dev);
4637         uint32_t temp;
4638
4639         temp = I915_READ(SOUTH_CHICKEN1);
4640         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4641                 return;
4642
4643         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4644         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4645
4646         temp &= ~FDI_BC_BIFURCATION_SELECT;
4647         if (enable)
4648                 temp |= FDI_BC_BIFURCATION_SELECT;
4649
4650         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4651         I915_WRITE(SOUTH_CHICKEN1, temp);
4652         POSTING_READ(SOUTH_CHICKEN1);
4653 }
4654
4655 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4656 {
4657         struct drm_device *dev = intel_crtc->base.dev;
4658
4659         switch (intel_crtc->pipe) {
4660         case PIPE_A:
4661                 break;
4662         case PIPE_B:
4663                 if (intel_crtc->config->fdi_lanes > 2)
4664                         cpt_set_fdi_bc_bifurcation(dev, false);
4665                 else
4666                         cpt_set_fdi_bc_bifurcation(dev, true);
4667
4668                 break;
4669         case PIPE_C:
4670                 cpt_set_fdi_bc_bifurcation(dev, true);
4671
4672                 break;
4673         default:
4674                 BUG();
4675         }
4676 }
4677
4678 /*
4679  * Finds the encoder associated with the given CRTC. This can only be
4680  * used when we know that the CRTC isn't feeding multiple encoders!
4681  */
4682 static struct intel_encoder *
4683 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
4684                            const struct intel_crtc_state *crtc_state)
4685 {
4686         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4687         const struct drm_connector_state *connector_state;
4688         const struct drm_connector *connector;
4689         struct intel_encoder *encoder = NULL;
4690         int num_encoders = 0;
4691         int i;
4692
4693         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4694                 if (connector_state->crtc != &crtc->base)
4695                         continue;
4696
4697                 encoder = to_intel_encoder(connector_state->best_encoder);
4698                 num_encoders++;
4699         }
4700
4701         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
4702              num_encoders, pipe_name(crtc->pipe));
4703
4704         return encoder;
4705 }
4706
4707 /*
4708  * Enable PCH resources required for PCH ports:
4709  *   - PCH PLLs
4710  *   - FDI training & RX/TX
4711  *   - update transcoder timings
4712  *   - DP transcoding bits
4713  *   - transcoder
4714  */
4715 static void ironlake_pch_enable(const struct intel_atomic_state *state,
4716                                 const struct intel_crtc_state *crtc_state)
4717 {
4718         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4719         struct drm_device *dev = crtc->base.dev;
4720         struct drm_i915_private *dev_priv = to_i915(dev);
4721         int pipe = crtc->pipe;
4722         u32 temp;
4723
4724         assert_pch_transcoder_disabled(dev_priv, pipe);
4725
4726         if (IS_IVYBRIDGE(dev_priv))
4727                 ivybridge_update_fdi_bc_bifurcation(crtc);
4728
4729         /* Write the TU size bits before fdi link training, so that error
4730          * detection works. */
4731         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4732                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4733
4734         /* For PCH output, training FDI link */
4735         dev_priv->display.fdi_link_train(crtc, crtc_state);
4736
4737         /* We need to program the right clock selection before writing the pixel
4738          * mutliplier into the DPLL. */
4739         if (HAS_PCH_CPT(dev_priv)) {
4740                 u32 sel;
4741
4742                 temp = I915_READ(PCH_DPLL_SEL);
4743                 temp |= TRANS_DPLL_ENABLE(pipe);
4744                 sel = TRANS_DPLLB_SEL(pipe);
4745                 if (crtc_state->shared_dpll ==
4746                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4747                         temp |= sel;
4748                 else
4749                         temp &= ~sel;
4750                 I915_WRITE(PCH_DPLL_SEL, temp);
4751         }
4752
4753         /* XXX: pch pll's can be enabled any time before we enable the PCH
4754          * transcoder, and we actually should do this to not upset any PCH
4755          * transcoder that already use the clock when we share it.
4756          *
4757          * Note that enable_shared_dpll tries to do the right thing, but
4758          * get_shared_dpll unconditionally resets the pll - we need that to have
4759          * the right LVDS enable sequence. */
4760         intel_enable_shared_dpll(crtc);
4761
4762         /* set transcoder timing, panel must allow it */
4763         assert_panel_unlocked(dev_priv, pipe);
4764         ironlake_pch_transcoder_set_timings(crtc, pipe);
4765
4766         intel_fdi_normal_train(crtc);
4767
4768         /* For PCH DP, enable TRANS_DP_CTL */
4769         if (HAS_PCH_CPT(dev_priv) &&
4770             intel_crtc_has_dp_encoder(crtc_state)) {
4771                 const struct drm_display_mode *adjusted_mode =
4772                         &crtc_state->base.adjusted_mode;
4773                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4774                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4775                 enum port port;
4776
4777                 temp = I915_READ(reg);
4778                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4779                           TRANS_DP_SYNC_MASK |
4780                           TRANS_DP_BPC_MASK);
4781                 temp |= TRANS_DP_OUTPUT_ENABLE;
4782                 temp |= bpc << 9; /* same format but at 11:9 */
4783
4784                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4785                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4786                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4787                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4788
4789                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
4790                 WARN_ON(port < PORT_B || port > PORT_D);
4791                 temp |= TRANS_DP_PORT_SEL(port);
4792
4793                 I915_WRITE(reg, temp);
4794         }
4795
4796         ironlake_enable_pch_transcoder(dev_priv, pipe);
4797 }
4798
4799 static void lpt_pch_enable(const struct intel_atomic_state *state,
4800                            const struct intel_crtc_state *crtc_state)
4801 {
4802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4803         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4804         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4805
4806         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
4807
4808         lpt_program_iclkip(crtc);
4809
4810         /* Set transcoder timing. */
4811         ironlake_pch_transcoder_set_timings(crtc, PIPE_A);
4812
4813         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4814 }
4815
4816 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4817 {
4818         struct drm_i915_private *dev_priv = to_i915(dev);
4819         i915_reg_t dslreg = PIPEDSL(pipe);
4820         u32 temp;
4821
4822         temp = I915_READ(dslreg);
4823         udelay(500);
4824         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4825                 if (wait_for(I915_READ(dslreg) != temp, 5))
4826                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4827         }
4828 }
4829
4830 /*
4831  * The hardware phase 0.0 refers to the center of the pixel.
4832  * We want to start from the top/left edge which is phase
4833  * -0.5. That matches how the hardware calculates the scaling
4834  * factors (from top-left of the first pixel to bottom-right
4835  * of the last pixel, as opposed to the pixel centers).
4836  *
4837  * For 4:2:0 subsampled chroma planes we obviously have to
4838  * adjust that so that the chroma sample position lands in
4839  * the right spot.
4840  *
4841  * Note that for packed YCbCr 4:2:2 formats there is no way to
4842  * control chroma siting. The hardware simply replicates the
4843  * chroma samples for both of the luma samples, and thus we don't
4844  * actually get the expected MPEG2 chroma siting convention :(
4845  * The same behaviour is observed on pre-SKL platforms as well.
4846  */
4847 u16 skl_scaler_calc_phase(int sub, bool chroma_cosited)
4848 {
4849         int phase = -0x8000;
4850         u16 trip = 0;
4851
4852         if (chroma_cosited)
4853                 phase += (sub - 1) * 0x8000 / sub;
4854
4855         if (phase < 0)
4856                 phase = 0x10000 + phase;
4857         else
4858                 trip = PS_PHASE_TRIP;
4859
4860         return ((phase >> 2) & PS_PHASE_MASK) | trip;
4861 }
4862
4863 static int
4864 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4865                   unsigned int scaler_user, int *scaler_id,
4866                   int src_w, int src_h, int dst_w, int dst_h,
4867                   bool plane_scaler_check,
4868                   uint32_t pixel_format)
4869 {
4870         struct intel_crtc_scaler_state *scaler_state =
4871                 &crtc_state->scaler_state;
4872         struct intel_crtc *intel_crtc =
4873                 to_intel_crtc(crtc_state->base.crtc);
4874         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4875         const struct drm_display_mode *adjusted_mode =
4876                 &crtc_state->base.adjusted_mode;
4877         int need_scaling;
4878
4879         /*
4880          * Src coordinates are already rotated by 270 degrees for
4881          * the 90/270 degree plane rotation cases (to match the
4882          * GTT mapping), hence no need to account for rotation here.
4883          */
4884         need_scaling = src_w != dst_w || src_h != dst_h;
4885
4886         if (plane_scaler_check)
4887                 if (pixel_format == DRM_FORMAT_NV12)
4888                         need_scaling = true;
4889
4890         if (crtc_state->ycbcr420 && scaler_user == SKL_CRTC_INDEX)
4891                 need_scaling = true;
4892
4893         /*
4894          * Scaling/fitting not supported in IF-ID mode in GEN9+
4895          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
4896          * Once NV12 is enabled, handle it here while allocating scaler
4897          * for NV12.
4898          */
4899         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
4900             need_scaling && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4901                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
4902                 return -EINVAL;
4903         }
4904
4905         /*
4906          * if plane is being disabled or scaler is no more required or force detach
4907          *  - free scaler binded to this plane/crtc
4908          *  - in order to do this, update crtc->scaler_usage
4909          *
4910          * Here scaler state in crtc_state is set free so that
4911          * scaler can be assigned to other user. Actual register
4912          * update to free the scaler is done in plane/panel-fit programming.
4913          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4914          */
4915         if (force_detach || !need_scaling) {
4916                 if (*scaler_id >= 0) {
4917                         scaler_state->scaler_users &= ~(1 << scaler_user);
4918                         scaler_state->scalers[*scaler_id].in_use = 0;
4919
4920                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4921                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4922                                 intel_crtc->pipe, scaler_user, *scaler_id,
4923                                 scaler_state->scaler_users);
4924                         *scaler_id = -1;
4925                 }
4926                 return 0;
4927         }
4928
4929         if (plane_scaler_check && pixel_format == DRM_FORMAT_NV12 &&
4930             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
4931                 DRM_DEBUG_KMS("NV12: src dimensions not met\n");
4932                 return -EINVAL;
4933         }
4934
4935         /* range checks */
4936         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4937             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4938             (IS_GEN11(dev_priv) &&
4939              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
4940               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
4941             (!IS_GEN11(dev_priv) &&
4942              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4943               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
4944                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4945                         "size is out of scaler range\n",
4946                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4947                 return -EINVAL;
4948         }
4949
4950         /* mark this plane as a scaler user in crtc_state */
4951         scaler_state->scaler_users |= (1 << scaler_user);
4952         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4953                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4954                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4955                 scaler_state->scaler_users);
4956
4957         return 0;
4958 }
4959
4960 /**
4961  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4962  *
4963  * @state: crtc's scaler state
4964  *
4965  * Return
4966  *     0 - scaler_usage updated successfully
4967  *    error - requested scaling cannot be supported or other error condition
4968  */
4969 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4970 {
4971         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4972
4973         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4974                                  &state->scaler_state.scaler_id,
4975                                  state->pipe_src_w, state->pipe_src_h,
4976                                  adjusted_mode->crtc_hdisplay,
4977                                  adjusted_mode->crtc_vdisplay, false, 0);
4978 }
4979
4980 /**
4981  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4982  * @crtc_state: crtc's scaler state
4983  * @plane_state: atomic plane state to update
4984  *
4985  * Return
4986  *     0 - scaler_usage updated successfully
4987  *    error - requested scaling cannot be supported or other error condition
4988  */
4989 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4990                                    struct intel_plane_state *plane_state)
4991 {
4992
4993         struct intel_plane *intel_plane =
4994                 to_intel_plane(plane_state->base.plane);
4995         struct drm_framebuffer *fb = plane_state->base.fb;
4996         int ret;
4997
4998         bool force_detach = !fb || !plane_state->base.visible;
4999
5000         ret = skl_update_scaler(crtc_state, force_detach,
5001                                 drm_plane_index(&intel_plane->base),
5002                                 &plane_state->scaler_id,
5003                                 drm_rect_width(&plane_state->base.src) >> 16,
5004                                 drm_rect_height(&plane_state->base.src) >> 16,
5005                                 drm_rect_width(&plane_state->base.dst),
5006                                 drm_rect_height(&plane_state->base.dst),
5007                                 fb ? true : false, fb ? fb->format->format : 0);
5008
5009         if (ret || plane_state->scaler_id < 0)
5010                 return ret;
5011
5012         /* check colorkey */
5013         if (plane_state->ckey.flags) {
5014                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5015                               intel_plane->base.base.id,
5016                               intel_plane->base.name);
5017                 return -EINVAL;
5018         }
5019
5020         /* Check src format */
5021         switch (fb->format->format) {
5022         case DRM_FORMAT_RGB565:
5023         case DRM_FORMAT_XBGR8888:
5024         case DRM_FORMAT_XRGB8888:
5025         case DRM_FORMAT_ABGR8888:
5026         case DRM_FORMAT_ARGB8888:
5027         case DRM_FORMAT_XRGB2101010:
5028         case DRM_FORMAT_XBGR2101010:
5029         case DRM_FORMAT_YUYV:
5030         case DRM_FORMAT_YVYU:
5031         case DRM_FORMAT_UYVY:
5032         case DRM_FORMAT_VYUY:
5033         case DRM_FORMAT_NV12:
5034                 break;
5035         default:
5036                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5037                               intel_plane->base.base.id, intel_plane->base.name,
5038                               fb->base.id, fb->format->format);
5039                 return -EINVAL;
5040         }
5041
5042         return 0;
5043 }
5044
5045 static void skylake_scaler_disable(struct intel_crtc *crtc)
5046 {
5047         int i;
5048
5049         for (i = 0; i < crtc->num_scalers; i++)
5050                 skl_detach_scaler(crtc, i);
5051 }
5052
5053 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5054 {
5055         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5056         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5057         enum pipe pipe = crtc->pipe;
5058         const struct intel_crtc_scaler_state *scaler_state =
5059                 &crtc_state->scaler_state;
5060
5061         if (crtc_state->pch_pfit.enabled) {
5062                 u16 uv_rgb_hphase, uv_rgb_vphase;
5063                 int id;
5064
5065                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5066                         return;
5067
5068                 uv_rgb_hphase = skl_scaler_calc_phase(1, false);
5069                 uv_rgb_vphase = skl_scaler_calc_phase(1, false);
5070
5071                 id = scaler_state->scaler_id;
5072                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5073                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5074                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5075                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5076                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5077                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5078                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5079                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5080         }
5081 }
5082
5083 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5084 {
5085         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5086         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5087         int pipe = crtc->pipe;
5088
5089         if (crtc_state->pch_pfit.enabled) {
5090                 /* Force use of hard-coded filter coefficients
5091                  * as some pre-programmed values are broken,
5092                  * e.g. x201.
5093                  */
5094                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5095                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5096                                                  PF_PIPE_SEL_IVB(pipe));
5097                 else
5098                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5099                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5100                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5101         }
5102 }
5103
5104 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5105 {
5106         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5107         struct drm_device *dev = crtc->base.dev;
5108         struct drm_i915_private *dev_priv = to_i915(dev);
5109
5110         if (!crtc_state->ips_enabled)
5111                 return;
5112
5113         /*
5114          * We can only enable IPS after we enable a plane and wait for a vblank
5115          * This function is called from post_plane_update, which is run after
5116          * a vblank wait.
5117          */
5118         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5119
5120         if (IS_BROADWELL(dev_priv)) {
5121                 mutex_lock(&dev_priv->pcu_lock);
5122                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5123                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5124                 mutex_unlock(&dev_priv->pcu_lock);
5125                 /* Quoting Art Runyan: "its not safe to expect any particular
5126                  * value in IPS_CTL bit 31 after enabling IPS through the
5127                  * mailbox." Moreover, the mailbox may return a bogus state,
5128                  * so we need to just enable it and continue on.
5129                  */
5130         } else {
5131                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5132                 /* The bit only becomes 1 in the next vblank, so this wait here
5133                  * is essentially intel_wait_for_vblank. If we don't have this
5134                  * and don't wait for vblanks until the end of crtc_enable, then
5135                  * the HW state readout code will complain that the expected
5136                  * IPS_CTL value is not the one we read. */
5137                 if (intel_wait_for_register(dev_priv,
5138                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5139                                             50))
5140                         DRM_ERROR("Timed out waiting for IPS enable\n");
5141         }
5142 }
5143
5144 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5145 {
5146         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5147         struct drm_device *dev = crtc->base.dev;
5148         struct drm_i915_private *dev_priv = to_i915(dev);
5149
5150         if (!crtc_state->ips_enabled)
5151                 return;
5152
5153         if (IS_BROADWELL(dev_priv)) {
5154                 mutex_lock(&dev_priv->pcu_lock);
5155                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5156                 mutex_unlock(&dev_priv->pcu_lock);
5157                 /*
5158                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5159                  * 42ms timeout value leads to occasional timeouts so use 100ms
5160                  * instead.
5161                  */
5162                 if (intel_wait_for_register(dev_priv,
5163                                             IPS_CTL, IPS_ENABLE, 0,
5164                                             100))
5165                         DRM_ERROR("Timed out waiting for IPS disable\n");
5166         } else {
5167                 I915_WRITE(IPS_CTL, 0);
5168                 POSTING_READ(IPS_CTL);
5169         }
5170
5171         /* We need to wait for a vblank before we can disable the plane. */
5172         intel_wait_for_vblank(dev_priv, crtc->pipe);
5173 }
5174
5175 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5176 {
5177         if (intel_crtc->overlay) {
5178                 struct drm_device *dev = intel_crtc->base.dev;
5179
5180                 mutex_lock(&dev->struct_mutex);
5181                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5182                 mutex_unlock(&dev->struct_mutex);
5183         }
5184
5185         /* Let userspace switch the overlay on again. In most cases userspace
5186          * has to recompute where to put it anyway.
5187          */
5188 }
5189
5190 /**
5191  * intel_post_enable_primary - Perform operations after enabling primary plane
5192  * @crtc: the CRTC whose primary plane was just enabled
5193  * @new_crtc_state: the enabling state
5194  *
5195  * Performs potentially sleeping operations that must be done after the primary
5196  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5197  * called due to an explicit primary plane update, or due to an implicit
5198  * re-enable that is caused when a sprite plane is updated to no longer
5199  * completely hide the primary plane.
5200  */
5201 static void
5202 intel_post_enable_primary(struct drm_crtc *crtc,
5203                           const struct intel_crtc_state *new_crtc_state)
5204 {
5205         struct drm_device *dev = crtc->dev;
5206         struct drm_i915_private *dev_priv = to_i915(dev);
5207         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5208         int pipe = intel_crtc->pipe;
5209
5210         /*
5211          * Gen2 reports pipe underruns whenever all planes are disabled.
5212          * So don't enable underrun reporting before at least some planes
5213          * are enabled.
5214          * FIXME: Need to fix the logic to work when we turn off all planes
5215          * but leave the pipe running.
5216          */
5217         if (IS_GEN2(dev_priv))
5218                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5219
5220         /* Underruns don't always raise interrupts, so check manually. */
5221         intel_check_cpu_fifo_underruns(dev_priv);
5222         intel_check_pch_fifo_underruns(dev_priv);
5223 }
5224
5225 /* FIXME get rid of this and use pre_plane_update */
5226 static void
5227 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5228 {
5229         struct drm_device *dev = crtc->dev;
5230         struct drm_i915_private *dev_priv = to_i915(dev);
5231         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5232         int pipe = intel_crtc->pipe;
5233
5234         /*
5235          * Gen2 reports pipe underruns whenever all planes are disabled.
5236          * So disable underrun reporting before all the planes get disabled.
5237          */
5238         if (IS_GEN2(dev_priv))
5239                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5240
5241         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5242
5243         /*
5244          * Vblank time updates from the shadow to live plane control register
5245          * are blocked if the memory self-refresh mode is active at that
5246          * moment. So to make sure the plane gets truly disabled, disable
5247          * first the self-refresh mode. The self-refresh enable bit in turn
5248          * will be checked/applied by the HW only at the next frame start
5249          * event which is after the vblank start event, so we need to have a
5250          * wait-for-vblank between disabling the plane and the pipe.
5251          */
5252         if (HAS_GMCH_DISPLAY(dev_priv) &&
5253             intel_set_memory_cxsr(dev_priv, false))
5254                 intel_wait_for_vblank(dev_priv, pipe);
5255 }
5256
5257 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5258                                        const struct intel_crtc_state *new_crtc_state)
5259 {
5260         if (!old_crtc_state->ips_enabled)
5261                 return false;
5262
5263         if (needs_modeset(&new_crtc_state->base))
5264                 return true;
5265
5266         return !new_crtc_state->ips_enabled;
5267 }
5268
5269 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5270                                        const struct intel_crtc_state *new_crtc_state)
5271 {
5272         if (!new_crtc_state->ips_enabled)
5273                 return false;
5274
5275         if (needs_modeset(&new_crtc_state->base))
5276                 return true;
5277
5278         /*
5279          * We can't read out IPS on broadwell, assume the worst and
5280          * forcibly enable IPS on the first fastset.
5281          */
5282         if (new_crtc_state->update_pipe &&
5283             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5284                 return true;
5285
5286         return !old_crtc_state->ips_enabled;
5287 }
5288
5289 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5290                           const struct intel_crtc_state *crtc_state)
5291 {
5292         if (!crtc_state->nv12_planes)
5293                 return false;
5294
5295         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
5296                 return false;
5297
5298         if ((INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv)) ||
5299             IS_CANNONLAKE(dev_priv))
5300                 return true;
5301
5302         return false;
5303 }
5304
5305 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5306 {
5307         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5308         struct drm_device *dev = crtc->base.dev;
5309         struct drm_i915_private *dev_priv = to_i915(dev);
5310         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5311         struct intel_crtc_state *pipe_config =
5312                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state),
5313                                                 crtc);
5314         struct drm_plane *primary = crtc->base.primary;
5315         struct drm_plane_state *old_primary_state =
5316                 drm_atomic_get_old_plane_state(old_state, primary);
5317
5318         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5319
5320         if (pipe_config->update_wm_post && pipe_config->base.active)
5321                 intel_update_watermarks(crtc);
5322
5323         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5324                 hsw_enable_ips(pipe_config);
5325
5326         if (old_primary_state) {
5327                 struct drm_plane_state *new_primary_state =
5328                         drm_atomic_get_new_plane_state(old_state, primary);
5329
5330                 intel_fbc_post_update(crtc);
5331
5332                 if (new_primary_state->visible &&
5333                     (needs_modeset(&pipe_config->base) ||
5334                      !old_primary_state->visible))
5335                         intel_post_enable_primary(&crtc->base, pipe_config);
5336         }
5337
5338         /* Display WA 827 */
5339         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5340             !needs_nv12_wa(dev_priv, pipe_config)) {
5341                 skl_wa_clkgate(dev_priv, crtc->pipe, false);
5342                 skl_wa_528(dev_priv, crtc->pipe, false);
5343         }
5344 }
5345
5346 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5347                                    struct intel_crtc_state *pipe_config)
5348 {
5349         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5350         struct drm_device *dev = crtc->base.dev;
5351         struct drm_i915_private *dev_priv = to_i915(dev);
5352         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5353         struct drm_plane *primary = crtc->base.primary;
5354         struct drm_plane_state *old_primary_state =
5355                 drm_atomic_get_old_plane_state(old_state, primary);
5356         bool modeset = needs_modeset(&pipe_config->base);
5357         struct intel_atomic_state *old_intel_state =
5358                 to_intel_atomic_state(old_state);
5359
5360         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5361                 hsw_disable_ips(old_crtc_state);
5362
5363         if (old_primary_state) {
5364                 struct intel_plane_state *new_primary_state =
5365                         intel_atomic_get_new_plane_state(old_intel_state,
5366                                                          to_intel_plane(primary));
5367
5368                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5369                 /*
5370                  * Gen2 reports pipe underruns whenever all planes are disabled.
5371                  * So disable underrun reporting before all the planes get disabled.
5372                  */
5373                 if (IS_GEN2(dev_priv) && old_primary_state->visible &&
5374                     (modeset || !new_primary_state->base.visible))
5375                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5376         }
5377
5378         /* Display WA 827 */
5379         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5380             needs_nv12_wa(dev_priv, pipe_config)) {
5381                 skl_wa_clkgate(dev_priv, crtc->pipe, true);
5382                 skl_wa_528(dev_priv, crtc->pipe, true);
5383         }
5384
5385         /*
5386          * Vblank time updates from the shadow to live plane control register
5387          * are blocked if the memory self-refresh mode is active at that
5388          * moment. So to make sure the plane gets truly disabled, disable
5389          * first the self-refresh mode. The self-refresh enable bit in turn
5390          * will be checked/applied by the HW only at the next frame start
5391          * event which is after the vblank start event, so we need to have a
5392          * wait-for-vblank between disabling the plane and the pipe.
5393          */
5394         if (HAS_GMCH_DISPLAY(dev_priv) && old_crtc_state->base.active &&
5395             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5396                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5397
5398         /*
5399          * IVB workaround: must disable low power watermarks for at least
5400          * one frame before enabling scaling.  LP watermarks can be re-enabled
5401          * when scaling is disabled.
5402          *
5403          * WaCxSRDisabledForSpriteScaling:ivb
5404          */
5405         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev))
5406                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5407
5408         /*
5409          * If we're doing a modeset, we're done.  No need to do any pre-vblank
5410          * watermark programming here.
5411          */
5412         if (needs_modeset(&pipe_config->base))
5413                 return;
5414
5415         /*
5416          * For platforms that support atomic watermarks, program the
5417          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
5418          * will be the intermediate values that are safe for both pre- and
5419          * post- vblank; when vblank happens, the 'active' values will be set
5420          * to the final 'target' values and we'll do this again to get the
5421          * optimal watermarks.  For gen9+ platforms, the values we program here
5422          * will be the final target values which will get automatically latched
5423          * at vblank time; no further programming will be necessary.
5424          *
5425          * If a platform hasn't been transitioned to atomic watermarks yet,
5426          * we'll continue to update watermarks the old way, if flags tell
5427          * us to.
5428          */
5429         if (dev_priv->display.initial_watermarks != NULL)
5430                 dev_priv->display.initial_watermarks(old_intel_state,
5431                                                      pipe_config);
5432         else if (pipe_config->update_wm_pre)
5433                 intel_update_watermarks(crtc);
5434 }
5435
5436 static void intel_crtc_disable_planes(struct intel_crtc *crtc, unsigned plane_mask)
5437 {
5438         struct drm_device *dev = crtc->base.dev;
5439         struct intel_plane *plane;
5440         unsigned fb_bits = 0;
5441
5442         intel_crtc_dpms_overlay_disable(crtc);
5443
5444         for_each_intel_plane_on_crtc(dev, crtc, plane) {
5445                 if (plane_mask & BIT(plane->id)) {
5446                         plane->disable_plane(plane, crtc);
5447
5448                         fb_bits |= plane->frontbuffer_bit;
5449                 }
5450         }
5451
5452         intel_frontbuffer_flip(to_i915(dev), fb_bits);
5453 }
5454
5455 static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc,
5456                                           struct intel_crtc_state *crtc_state,
5457                                           struct drm_atomic_state *old_state)
5458 {
5459         struct drm_connector_state *conn_state;
5460         struct drm_connector *conn;
5461         int i;
5462
5463         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5464                 struct intel_encoder *encoder =
5465                         to_intel_encoder(conn_state->best_encoder);
5466
5467                 if (conn_state->crtc != crtc)
5468                         continue;
5469
5470                 if (encoder->pre_pll_enable)
5471                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
5472         }
5473 }
5474
5475 static void intel_encoders_pre_enable(struct drm_crtc *crtc,
5476                                       struct intel_crtc_state *crtc_state,
5477                                       struct drm_atomic_state *old_state)
5478 {
5479         struct drm_connector_state *conn_state;
5480         struct drm_connector *conn;
5481         int i;
5482
5483         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5484                 struct intel_encoder *encoder =
5485                         to_intel_encoder(conn_state->best_encoder);
5486
5487                 if (conn_state->crtc != crtc)
5488                         continue;
5489
5490                 if (encoder->pre_enable)
5491                         encoder->pre_enable(encoder, crtc_state, conn_state);
5492         }
5493 }
5494
5495 static void intel_encoders_enable(struct drm_crtc *crtc,
5496                                   struct intel_crtc_state *crtc_state,
5497                                   struct drm_atomic_state *old_state)
5498 {
5499         struct drm_connector_state *conn_state;
5500         struct drm_connector *conn;
5501         int i;
5502
5503         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5504                 struct intel_encoder *encoder =
5505                         to_intel_encoder(conn_state->best_encoder);
5506
5507                 if (conn_state->crtc != crtc)
5508                         continue;
5509
5510                 encoder->enable(encoder, crtc_state, conn_state);
5511                 intel_opregion_notify_encoder(encoder, true);
5512         }
5513 }
5514
5515 static void intel_encoders_disable(struct drm_crtc *crtc,
5516                                    struct intel_crtc_state *old_crtc_state,
5517                                    struct drm_atomic_state *old_state)
5518 {
5519         struct drm_connector_state *old_conn_state;
5520         struct drm_connector *conn;
5521         int i;
5522
5523         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5524                 struct intel_encoder *encoder =
5525                         to_intel_encoder(old_conn_state->best_encoder);
5526
5527                 if (old_conn_state->crtc != crtc)
5528                         continue;
5529
5530                 intel_opregion_notify_encoder(encoder, false);
5531                 encoder->disable(encoder, old_crtc_state, old_conn_state);
5532         }
5533 }
5534
5535 static void intel_encoders_post_disable(struct drm_crtc *crtc,
5536                                         struct intel_crtc_state *old_crtc_state,
5537                                         struct drm_atomic_state *old_state)
5538 {
5539         struct drm_connector_state *old_conn_state;
5540         struct drm_connector *conn;
5541         int i;
5542
5543         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5544                 struct intel_encoder *encoder =
5545                         to_intel_encoder(old_conn_state->best_encoder);
5546
5547                 if (old_conn_state->crtc != crtc)
5548                         continue;
5549
5550                 if (encoder->post_disable)
5551                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
5552         }
5553 }
5554
5555 static void intel_encoders_post_pll_disable(struct drm_crtc *crtc,
5556                                             struct intel_crtc_state *old_crtc_state,
5557                                             struct drm_atomic_state *old_state)
5558 {
5559         struct drm_connector_state *old_conn_state;
5560         struct drm_connector *conn;
5561         int i;
5562
5563         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5564                 struct intel_encoder *encoder =
5565                         to_intel_encoder(old_conn_state->best_encoder);
5566
5567                 if (old_conn_state->crtc != crtc)
5568                         continue;
5569
5570                 if (encoder->post_pll_disable)
5571                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
5572         }
5573 }
5574
5575 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5576                                  struct drm_atomic_state *old_state)
5577 {
5578         struct drm_crtc *crtc = pipe_config->base.crtc;
5579         struct drm_device *dev = crtc->dev;
5580         struct drm_i915_private *dev_priv = to_i915(dev);
5581         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5582         int pipe = intel_crtc->pipe;
5583         struct intel_atomic_state *old_intel_state =
5584                 to_intel_atomic_state(old_state);
5585
5586         if (WARN_ON(intel_crtc->active))
5587                 return;
5588
5589         /*
5590          * Sometimes spurious CPU pipe underruns happen during FDI
5591          * training, at least with VGA+HDMI cloning. Suppress them.
5592          *
5593          * On ILK we get an occasional spurious CPU pipe underruns
5594          * between eDP port A enable and vdd enable. Also PCH port
5595          * enable seems to result in the occasional CPU pipe underrun.
5596          *
5597          * Spurious PCH underruns also occur during PCH enabling.
5598          */
5599         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5600         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5601
5602         if (intel_crtc->config->has_pch_encoder)
5603                 intel_prepare_shared_dpll(intel_crtc);
5604
5605         if (intel_crtc_has_dp_encoder(intel_crtc->config))
5606                 intel_dp_set_m_n(intel_crtc, M1_N1);
5607
5608         intel_set_pipe_timings(intel_crtc);
5609         intel_set_pipe_src_size(intel_crtc);
5610
5611         if (intel_crtc->config->has_pch_encoder) {
5612                 intel_cpu_transcoder_set_m_n(intel_crtc,
5613                                      &intel_crtc->config->fdi_m_n, NULL);
5614         }
5615
5616         ironlake_set_pipeconf(pipe_config);
5617
5618         intel_crtc->active = true;
5619
5620         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5621
5622         if (intel_crtc->config->has_pch_encoder) {
5623                 /* Note: FDI PLL enabling _must_ be done before we enable the
5624                  * cpu pipes, hence this is separate from all the other fdi/pch
5625                  * enabling. */
5626                 ironlake_fdi_pll_enable(intel_crtc);
5627         } else {
5628                 assert_fdi_tx_disabled(dev_priv, pipe);
5629                 assert_fdi_rx_disabled(dev_priv, pipe);
5630         }
5631
5632         ironlake_pfit_enable(pipe_config);
5633
5634         /*
5635          * On ILK+ LUT must be loaded before the pipe is running but with
5636          * clocks enabled
5637          */
5638         intel_color_load_luts(&pipe_config->base);
5639
5640         if (dev_priv->display.initial_watermarks != NULL)
5641                 dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config);
5642         intel_enable_pipe(pipe_config);
5643
5644         if (intel_crtc->config->has_pch_encoder)
5645                 ironlake_pch_enable(old_intel_state, pipe_config);
5646
5647         assert_vblank_disabled(crtc);
5648         drm_crtc_vblank_on(crtc);
5649
5650         intel_encoders_enable(crtc, pipe_config, old_state);
5651
5652         if (HAS_PCH_CPT(dev_priv))
5653                 cpt_verify_modeset(dev, intel_crtc->pipe);
5654
5655         /*
5656          * Must wait for vblank to avoid spurious PCH FIFO underruns.
5657          * And a second vblank wait is needed at least on ILK with
5658          * some interlaced HDMI modes. Let's do the double wait always
5659          * in case there are more corner cases we don't know about.
5660          */
5661         if (intel_crtc->config->has_pch_encoder) {
5662                 intel_wait_for_vblank(dev_priv, pipe);
5663                 intel_wait_for_vblank(dev_priv, pipe);
5664         }
5665         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5666         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5667 }
5668
5669 /* IPS only exists on ULT machines and is tied to pipe A. */
5670 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
5671 {
5672         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
5673 }
5674
5675 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
5676                                             enum pipe pipe, bool apply)
5677 {
5678         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
5679         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
5680
5681         if (apply)
5682                 val |= mask;
5683         else
5684                 val &= ~mask;
5685
5686         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
5687 }
5688
5689 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
5690 {
5691         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5692         enum pipe pipe = crtc->pipe;
5693         uint32_t val;
5694
5695         val = MBUS_DBOX_BW_CREDIT(1) | MBUS_DBOX_A_CREDIT(2);
5696
5697         /* Program B credit equally to all pipes */
5698         val |= MBUS_DBOX_B_CREDIT(24 / INTEL_INFO(dev_priv)->num_pipes);
5699
5700         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
5701 }
5702
5703 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5704                                 struct drm_atomic_state *old_state)
5705 {
5706         struct drm_crtc *crtc = pipe_config->base.crtc;
5707         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5708         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5709         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
5710         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5711         struct intel_atomic_state *old_intel_state =
5712                 to_intel_atomic_state(old_state);
5713         bool psl_clkgate_wa;
5714         u32 pipe_chicken;
5715
5716         if (WARN_ON(intel_crtc->active))
5717                 return;
5718
5719         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
5720
5721         if (intel_crtc->config->shared_dpll)
5722                 intel_enable_shared_dpll(intel_crtc);
5723
5724         if (INTEL_GEN(dev_priv) >= 11)
5725                 icl_map_plls_to_ports(crtc, pipe_config, old_state);
5726
5727         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5728
5729         if (intel_crtc_has_dp_encoder(intel_crtc->config))
5730                 intel_dp_set_m_n(intel_crtc, M1_N1);
5731
5732         if (!transcoder_is_dsi(cpu_transcoder))
5733                 intel_set_pipe_timings(intel_crtc);
5734
5735         intel_set_pipe_src_size(intel_crtc);
5736
5737         if (cpu_transcoder != TRANSCODER_EDP &&
5738             !transcoder_is_dsi(cpu_transcoder)) {
5739                 I915_WRITE(PIPE_MULT(cpu_transcoder),
5740                            intel_crtc->config->pixel_multiplier - 1);
5741         }
5742
5743         if (intel_crtc->config->has_pch_encoder) {
5744                 intel_cpu_transcoder_set_m_n(intel_crtc,
5745                                      &intel_crtc->config->fdi_m_n, NULL);
5746         }
5747
5748         if (!transcoder_is_dsi(cpu_transcoder))
5749                 haswell_set_pipeconf(pipe_config);
5750
5751         haswell_set_pipemisc(pipe_config);
5752
5753         intel_color_set_csc(&pipe_config->base);
5754
5755         intel_crtc->active = true;
5756
5757         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
5758         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
5759                          intel_crtc->config->pch_pfit.enabled;
5760         if (psl_clkgate_wa)
5761                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
5762
5763         if (INTEL_GEN(dev_priv) >= 9)
5764                 skylake_pfit_enable(pipe_config);
5765         else
5766                 ironlake_pfit_enable(pipe_config);
5767
5768         /*
5769          * On ILK+ LUT must be loaded before the pipe is running but with
5770          * clocks enabled
5771          */
5772         intel_color_load_luts(&pipe_config->base);
5773
5774         /*
5775          * Display WA #1153: enable hardware to bypass the alpha math
5776          * and rounding for per-pixel values 00 and 0xff
5777          */
5778         if (INTEL_GEN(dev_priv) >= 11) {
5779                 pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
5780                 if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
5781                         I915_WRITE_FW(PIPE_CHICKEN(pipe),
5782                                       pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
5783         }
5784
5785         intel_ddi_set_pipe_settings(pipe_config);
5786         if (!transcoder_is_dsi(cpu_transcoder))
5787                 intel_ddi_enable_transcoder_func(pipe_config);
5788
5789         if (dev_priv->display.initial_watermarks != NULL)
5790                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5791
5792         if (INTEL_GEN(dev_priv) >= 11)
5793                 icl_pipe_mbus_enable(intel_crtc);
5794
5795         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5796         if (!transcoder_is_dsi(cpu_transcoder))
5797                 intel_enable_pipe(pipe_config);
5798
5799         if (intel_crtc->config->has_pch_encoder)
5800                 lpt_pch_enable(old_intel_state, pipe_config);
5801
5802         if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
5803                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
5804
5805         assert_vblank_disabled(crtc);
5806         drm_crtc_vblank_on(crtc);
5807
5808         intel_encoders_enable(crtc, pipe_config, old_state);
5809
5810         if (psl_clkgate_wa) {
5811                 intel_wait_for_vblank(dev_priv, pipe);
5812                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
5813         }
5814
5815         /* If we change the relative order between pipe/planes enabling, we need
5816          * to change the workaround. */
5817         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5818         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
5819                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5820                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5821         }
5822 }
5823
5824 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
5825 {
5826         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5827         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5828         enum pipe pipe = crtc->pipe;
5829
5830         /* To avoid upsetting the power well on haswell only disable the pfit if
5831          * it's in use. The hw state code will make sure we get this right. */
5832         if (old_crtc_state->pch_pfit.enabled) {
5833                 I915_WRITE(PF_CTL(pipe), 0);
5834                 I915_WRITE(PF_WIN_POS(pipe), 0);
5835                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5836         }
5837 }
5838
5839 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
5840                                   struct drm_atomic_state *old_state)
5841 {
5842         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5843         struct drm_device *dev = crtc->dev;
5844         struct drm_i915_private *dev_priv = to_i915(dev);
5845         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5846         int pipe = intel_crtc->pipe;
5847
5848         /*
5849          * Sometimes spurious CPU pipe underruns happen when the
5850          * pipe is already disabled, but FDI RX/TX is still enabled.
5851          * Happens at least with VGA+HDMI cloning. Suppress them.
5852          */
5853         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5854         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5855
5856         intel_encoders_disable(crtc, old_crtc_state, old_state);
5857
5858         drm_crtc_vblank_off(crtc);
5859         assert_vblank_disabled(crtc);
5860
5861         intel_disable_pipe(old_crtc_state);
5862
5863         ironlake_pfit_disable(old_crtc_state);
5864
5865         if (intel_crtc->config->has_pch_encoder)
5866                 ironlake_fdi_disable(crtc);
5867
5868         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5869
5870         if (intel_crtc->config->has_pch_encoder) {
5871                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5872
5873                 if (HAS_PCH_CPT(dev_priv)) {
5874                         i915_reg_t reg;
5875                         u32 temp;
5876
5877                         /* disable TRANS_DP_CTL */
5878                         reg = TRANS_DP_CTL(pipe);
5879                         temp = I915_READ(reg);
5880                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5881                                   TRANS_DP_PORT_SEL_MASK);
5882                         temp |= TRANS_DP_PORT_SEL_NONE;
5883                         I915_WRITE(reg, temp);
5884
5885                         /* disable DPLL_SEL */
5886                         temp = I915_READ(PCH_DPLL_SEL);
5887                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5888                         I915_WRITE(PCH_DPLL_SEL, temp);
5889                 }
5890
5891                 ironlake_fdi_pll_disable(intel_crtc);
5892         }
5893
5894         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5895         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5896 }
5897
5898 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5899                                  struct drm_atomic_state *old_state)
5900 {
5901         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5902         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5903         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5904         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
5905
5906         intel_encoders_disable(crtc, old_crtc_state, old_state);
5907
5908         drm_crtc_vblank_off(crtc);
5909         assert_vblank_disabled(crtc);
5910
5911         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5912         if (!transcoder_is_dsi(cpu_transcoder))
5913                 intel_disable_pipe(old_crtc_state);
5914
5915         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
5916                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
5917
5918         if (!transcoder_is_dsi(cpu_transcoder))
5919                 intel_ddi_disable_transcoder_func(old_crtc_state);
5920
5921         if (INTEL_GEN(dev_priv) >= 9)
5922                 skylake_scaler_disable(intel_crtc);
5923         else
5924                 ironlake_pfit_disable(old_crtc_state);
5925
5926         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5927
5928         if (INTEL_GEN(dev_priv) >= 11)
5929                 icl_unmap_plls_to_ports(crtc, old_crtc_state, old_state);
5930 }
5931
5932 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
5933 {
5934         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5935         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5936
5937         if (!crtc_state->gmch_pfit.control)
5938                 return;
5939
5940         /*
5941          * The panel fitter should only be adjusted whilst the pipe is disabled,
5942          * according to register description and PRM.
5943          */
5944         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5945         assert_pipe_disabled(dev_priv, crtc->pipe);
5946
5947         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
5948         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
5949
5950         /* Border color in case we don't scale up to the full screen. Black by
5951          * default, change to something else for debugging. */
5952         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5953 }
5954
5955 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
5956 {
5957         if (IS_ICELAKE(dev_priv))
5958                 return port >= PORT_C && port <= PORT_F;
5959
5960         return false;
5961 }
5962
5963 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
5964 {
5965         if (!intel_port_is_tc(dev_priv, port))
5966                 return PORT_TC_NONE;
5967
5968         return port - PORT_C;
5969 }
5970
5971 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
5972 {
5973         switch (port) {
5974         case PORT_A:
5975                 return POWER_DOMAIN_PORT_DDI_A_LANES;
5976         case PORT_B:
5977                 return POWER_DOMAIN_PORT_DDI_B_LANES;
5978         case PORT_C:
5979                 return POWER_DOMAIN_PORT_DDI_C_LANES;
5980         case PORT_D:
5981                 return POWER_DOMAIN_PORT_DDI_D_LANES;
5982         case PORT_E:
5983                 return POWER_DOMAIN_PORT_DDI_E_LANES;
5984         case PORT_F:
5985                 return POWER_DOMAIN_PORT_DDI_F_LANES;
5986         default:
5987                 MISSING_CASE(port);
5988                 return POWER_DOMAIN_PORT_OTHER;
5989         }
5990 }
5991
5992 static u64 get_crtc_power_domains(struct drm_crtc *crtc,
5993                                   struct intel_crtc_state *crtc_state)
5994 {
5995         struct drm_device *dev = crtc->dev;
5996         struct drm_i915_private *dev_priv = to_i915(dev);
5997         struct drm_encoder *encoder;
5998         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5999         enum pipe pipe = intel_crtc->pipe;
6000         u64 mask;
6001         enum transcoder transcoder = crtc_state->cpu_transcoder;
6002
6003         if (!crtc_state->base.active)
6004                 return 0;
6005
6006         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6007         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6008         if (crtc_state->pch_pfit.enabled ||
6009             crtc_state->pch_pfit.force_thru)
6010                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6011
6012         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
6013                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6014
6015                 mask |= BIT_ULL(intel_encoder->power_domain);
6016         }
6017
6018         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6019                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6020
6021         if (crtc_state->shared_dpll)
6022                 mask |= BIT_ULL(POWER_DOMAIN_PLLS);
6023
6024         return mask;
6025 }
6026
6027 static u64
6028 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
6029                                struct intel_crtc_state *crtc_state)
6030 {
6031         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6032         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6033         enum intel_display_power_domain domain;
6034         u64 domains, new_domains, old_domains;
6035
6036         old_domains = intel_crtc->enabled_power_domains;
6037         intel_crtc->enabled_power_domains = new_domains =
6038                 get_crtc_power_domains(crtc, crtc_state);
6039
6040         domains = new_domains & ~old_domains;
6041
6042         for_each_power_domain(domain, domains)
6043                 intel_display_power_get(dev_priv, domain);
6044
6045         return old_domains & ~new_domains;
6046 }
6047
6048 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6049                                       u64 domains)
6050 {
6051         enum intel_display_power_domain domain;
6052
6053         for_each_power_domain(domain, domains)
6054                 intel_display_power_put(dev_priv, domain);
6055 }
6056
6057 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6058                                    struct drm_atomic_state *old_state)
6059 {
6060         struct intel_atomic_state *old_intel_state =
6061                 to_intel_atomic_state(old_state);
6062         struct drm_crtc *crtc = pipe_config->base.crtc;
6063         struct drm_device *dev = crtc->dev;
6064         struct drm_i915_private *dev_priv = to_i915(dev);
6065         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6066         int pipe = intel_crtc->pipe;
6067
6068         if (WARN_ON(intel_crtc->active))
6069                 return;
6070
6071         if (intel_crtc_has_dp_encoder(intel_crtc->config))
6072                 intel_dp_set_m_n(intel_crtc, M1_N1);
6073
6074         intel_set_pipe_timings(intel_crtc);
6075         intel_set_pipe_src_size(intel_crtc);
6076
6077         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6078                 struct drm_i915_private *dev_priv = to_i915(dev);
6079
6080                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6081                 I915_WRITE(CHV_CANVAS(pipe), 0);
6082         }
6083
6084         i9xx_set_pipeconf(pipe_config);
6085
6086         intel_color_set_csc(&pipe_config->base);
6087
6088         intel_crtc->active = true;
6089
6090         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6091
6092         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
6093
6094         if (IS_CHERRYVIEW(dev_priv)) {
6095                 chv_prepare_pll(intel_crtc, intel_crtc->config);
6096                 chv_enable_pll(intel_crtc, intel_crtc->config);
6097         } else {
6098                 vlv_prepare_pll(intel_crtc, intel_crtc->config);
6099                 vlv_enable_pll(intel_crtc, intel_crtc->config);
6100         }
6101
6102         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6103
6104         i9xx_pfit_enable(pipe_config);
6105
6106         intel_color_load_luts(&pipe_config->base);
6107
6108         dev_priv->display.initial_watermarks(old_intel_state,
6109                                              pipe_config);
6110         intel_enable_pipe(pipe_config);
6111
6112         assert_vblank_disabled(crtc);
6113         drm_crtc_vblank_on(crtc);
6114
6115         intel_encoders_enable(crtc, pipe_config, old_state);
6116 }
6117
6118 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6119 {
6120         struct drm_device *dev = crtc->base.dev;
6121         struct drm_i915_private *dev_priv = to_i915(dev);
6122
6123         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6124         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6125 }
6126
6127 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6128                              struct drm_atomic_state *old_state)
6129 {
6130         struct intel_atomic_state *old_intel_state =
6131                 to_intel_atomic_state(old_state);
6132         struct drm_crtc *crtc = pipe_config->base.crtc;
6133         struct drm_device *dev = crtc->dev;
6134         struct drm_i915_private *dev_priv = to_i915(dev);
6135         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6136         enum pipe pipe = intel_crtc->pipe;
6137
6138         if (WARN_ON(intel_crtc->active))
6139                 return;
6140
6141         i9xx_set_pll_dividers(intel_crtc);
6142
6143         if (intel_crtc_has_dp_encoder(intel_crtc->config))
6144                 intel_dp_set_m_n(intel_crtc, M1_N1);
6145
6146         intel_set_pipe_timings(intel_crtc);
6147         intel_set_pipe_src_size(intel_crtc);
6148
6149         i9xx_set_pipeconf(pipe_config);
6150
6151         intel_crtc->active = true;
6152
6153         if (!IS_GEN2(dev_priv))
6154                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6155
6156         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6157
6158         i9xx_enable_pll(intel_crtc, pipe_config);
6159
6160         i9xx_pfit_enable(pipe_config);
6161
6162         intel_color_load_luts(&pipe_config->base);
6163
6164         if (dev_priv->display.initial_watermarks != NULL)
6165                 dev_priv->display.initial_watermarks(old_intel_state,
6166                                                      intel_crtc->config);
6167         else
6168                 intel_update_watermarks(intel_crtc);
6169         intel_enable_pipe(pipe_config);
6170
6171         assert_vblank_disabled(crtc);
6172         drm_crtc_vblank_on(crtc);
6173
6174         intel_encoders_enable(crtc, pipe_config, old_state);
6175 }
6176
6177 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6178 {
6179         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6180         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6181
6182         if (!old_crtc_state->gmch_pfit.control)
6183                 return;
6184
6185         assert_pipe_disabled(dev_priv, crtc->pipe);
6186
6187         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6188                       I915_READ(PFIT_CONTROL));
6189         I915_WRITE(PFIT_CONTROL, 0);
6190 }
6191
6192 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6193                               struct drm_atomic_state *old_state)
6194 {
6195         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6196         struct drm_device *dev = crtc->dev;
6197         struct drm_i915_private *dev_priv = to_i915(dev);
6198         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6199         int pipe = intel_crtc->pipe;
6200
6201         /*
6202          * On gen2 planes are double buffered but the pipe isn't, so we must
6203          * wait for planes to fully turn off before disabling the pipe.
6204          */
6205         if (IS_GEN2(dev_priv))
6206                 intel_wait_for_vblank(dev_priv, pipe);
6207
6208         intel_encoders_disable(crtc, old_crtc_state, old_state);
6209
6210         drm_crtc_vblank_off(crtc);
6211         assert_vblank_disabled(crtc);
6212
6213         intel_disable_pipe(old_crtc_state);
6214
6215         i9xx_pfit_disable(old_crtc_state);
6216
6217         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
6218
6219         if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
6220                 if (IS_CHERRYVIEW(dev_priv))
6221                         chv_disable_pll(dev_priv, pipe);
6222                 else if (IS_VALLEYVIEW(dev_priv))
6223                         vlv_disable_pll(dev_priv, pipe);
6224                 else
6225                         i9xx_disable_pll(intel_crtc);
6226         }
6227
6228         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
6229
6230         if (!IS_GEN2(dev_priv))
6231                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6232
6233         if (!dev_priv->display.initial_watermarks)
6234                 intel_update_watermarks(intel_crtc);
6235
6236         /* clock the pipe down to 640x480@60 to potentially save power */
6237         if (IS_I830(dev_priv))
6238                 i830_enable_pipe(dev_priv, pipe);
6239 }
6240
6241 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
6242                                         struct drm_modeset_acquire_ctx *ctx)
6243 {
6244         struct intel_encoder *encoder;
6245         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6246         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6247         enum intel_display_power_domain domain;
6248         struct intel_plane *plane;
6249         u64 domains;
6250         struct drm_atomic_state *state;
6251         struct intel_crtc_state *crtc_state;
6252         int ret;
6253
6254         if (!intel_crtc->active)
6255                 return;
6256
6257         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
6258                 const struct intel_plane_state *plane_state =
6259                         to_intel_plane_state(plane->base.state);
6260
6261                 if (plane_state->base.visible)
6262                         intel_plane_disable_noatomic(intel_crtc, plane);
6263         }
6264
6265         state = drm_atomic_state_alloc(crtc->dev);
6266         if (!state) {
6267                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
6268                               crtc->base.id, crtc->name);
6269                 return;
6270         }
6271
6272         state->acquire_ctx = ctx;
6273
6274         /* Everything's already locked, -EDEADLK can't happen. */
6275         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6276         ret = drm_atomic_add_affected_connectors(state, crtc);
6277
6278         WARN_ON(IS_ERR(crtc_state) || ret);
6279
6280         dev_priv->display.crtc_disable(crtc_state, state);
6281
6282         drm_atomic_state_put(state);
6283
6284         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6285                       crtc->base.id, crtc->name);
6286
6287         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
6288         crtc->state->active = false;
6289         intel_crtc->active = false;
6290         crtc->enabled = false;
6291         crtc->state->connector_mask = 0;
6292         crtc->state->encoder_mask = 0;
6293
6294         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
6295                 encoder->base.crtc = NULL;
6296
6297         intel_fbc_disable(intel_crtc);
6298         intel_update_watermarks(intel_crtc);
6299         intel_disable_shared_dpll(intel_crtc);
6300
6301         domains = intel_crtc->enabled_power_domains;
6302         for_each_power_domain(domain, domains)
6303                 intel_display_power_put(dev_priv, domain);
6304         intel_crtc->enabled_power_domains = 0;
6305
6306         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
6307         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
6308         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
6309 }
6310
6311 /*
6312  * turn all crtc's off, but do not adjust state
6313  * This has to be paired with a call to intel_modeset_setup_hw_state.
6314  */
6315 int intel_display_suspend(struct drm_device *dev)
6316 {
6317         struct drm_i915_private *dev_priv = to_i915(dev);
6318         struct drm_atomic_state *state;
6319         int ret;
6320
6321         state = drm_atomic_helper_suspend(dev);
6322         ret = PTR_ERR_OR_ZERO(state);
6323         if (ret)
6324                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6325         else
6326                 dev_priv->modeset_restore_state = state;
6327         return ret;
6328 }
6329
6330 void intel_encoder_destroy(struct drm_encoder *encoder)
6331 {
6332         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6333
6334         drm_encoder_cleanup(encoder);
6335         kfree(intel_encoder);
6336 }
6337
6338 /* Cross check the actual hw state with our own modeset state tracking (and it's
6339  * internal consistency). */
6340 static void intel_connector_verify_state(struct drm_crtc_state *crtc_state,
6341                                          struct drm_connector_state *conn_state)
6342 {
6343         struct intel_connector *connector = to_intel_connector(conn_state->connector);
6344
6345         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6346                       connector->base.base.id,
6347                       connector->base.name);
6348
6349         if (connector->get_hw_state(connector)) {
6350                 struct intel_encoder *encoder = connector->encoder;
6351
6352                 I915_STATE_WARN(!crtc_state,
6353                          "connector enabled without attached crtc\n");
6354
6355                 if (!crtc_state)
6356                         return;
6357
6358                 I915_STATE_WARN(!crtc_state->active,
6359                       "connector is active, but attached crtc isn't\n");
6360
6361                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6362                         return;
6363
6364                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6365                         "atomic encoder doesn't match attached encoder\n");
6366
6367                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6368                         "attached encoder crtc differs from connector crtc\n");
6369         } else {
6370                 I915_STATE_WARN(crtc_state && crtc_state->active,
6371                         "attached crtc is active, but connector isn't\n");
6372                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
6373                         "best encoder set without crtc!\n");
6374         }
6375 }
6376
6377 int intel_connector_init(struct intel_connector *connector)
6378 {
6379         struct intel_digital_connector_state *conn_state;
6380
6381         /*
6382          * Allocate enough memory to hold intel_digital_connector_state,
6383          * This might be a few bytes too many, but for connectors that don't
6384          * need it we'll free the state and allocate a smaller one on the first
6385          * succesful commit anyway.
6386          */
6387         conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL);
6388         if (!conn_state)
6389                 return -ENOMEM;
6390
6391         __drm_atomic_helper_connector_reset(&connector->base,
6392                                             &conn_state->base);
6393
6394         return 0;
6395 }
6396
6397 struct intel_connector *intel_connector_alloc(void)
6398 {
6399         struct intel_connector *connector;
6400
6401         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6402         if (!connector)
6403                 return NULL;
6404
6405         if (intel_connector_init(connector) < 0) {
6406                 kfree(connector);
6407                 return NULL;
6408         }
6409
6410         return connector;
6411 }
6412
6413 /*
6414  * Free the bits allocated by intel_connector_alloc.
6415  * This should only be used after intel_connector_alloc has returned
6416  * successfully, and before drm_connector_init returns successfully.
6417  * Otherwise the destroy callbacks for the connector and the state should
6418  * take care of proper cleanup/free
6419  */
6420 void intel_connector_free(struct intel_connector *connector)
6421 {
6422         kfree(to_intel_digital_connector_state(connector->base.state));
6423         kfree(connector);
6424 }
6425
6426 /* Simple connector->get_hw_state implementation for encoders that support only
6427  * one connector and no cloning and hence the encoder state determines the state
6428  * of the connector. */
6429 bool intel_connector_get_hw_state(struct intel_connector *connector)
6430 {
6431         enum pipe pipe = 0;
6432         struct intel_encoder *encoder = connector->encoder;
6433
6434         return encoder->get_hw_state(encoder, &pipe);
6435 }
6436
6437 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6438 {
6439         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6440                 return crtc_state->fdi_lanes;
6441
6442         return 0;
6443 }
6444
6445 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6446                                      struct intel_crtc_state *pipe_config)
6447 {
6448         struct drm_i915_private *dev_priv = to_i915(dev);
6449         struct drm_atomic_state *state = pipe_config->base.state;
6450         struct intel_crtc *other_crtc;
6451         struct intel_crtc_state *other_crtc_state;
6452
6453         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6454                       pipe_name(pipe), pipe_config->fdi_lanes);
6455         if (pipe_config->fdi_lanes > 4) {
6456                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6457                               pipe_name(pipe), pipe_config->fdi_lanes);
6458                 return -EINVAL;
6459         }
6460
6461         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
6462                 if (pipe_config->fdi_lanes > 2) {
6463                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6464                                       pipe_config->fdi_lanes);
6465                         return -EINVAL;
6466                 } else {
6467                         return 0;
6468                 }
6469         }
6470
6471         if (INTEL_INFO(dev_priv)->num_pipes == 2)
6472                 return 0;
6473
6474         /* Ivybridge 3 pipe is really complicated */
6475         switch (pipe) {
6476         case PIPE_A:
6477                 return 0;
6478         case PIPE_B:
6479                 if (pipe_config->fdi_lanes <= 2)
6480                         return 0;
6481
6482                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
6483                 other_crtc_state =
6484                         intel_atomic_get_crtc_state(state, other_crtc);
6485                 if (IS_ERR(other_crtc_state))
6486                         return PTR_ERR(other_crtc_state);
6487
6488                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6489                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6490                                       pipe_name(pipe), pipe_config->fdi_lanes);
6491                         return -EINVAL;
6492                 }
6493                 return 0;
6494         case PIPE_C:
6495                 if (pipe_config->fdi_lanes > 2) {
6496                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6497                                       pipe_name(pipe), pipe_config->fdi_lanes);
6498                         return -EINVAL;
6499                 }
6500
6501                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
6502                 other_crtc_state =
6503                         intel_atomic_get_crtc_state(state, other_crtc);
6504                 if (IS_ERR(other_crtc_state))
6505                         return PTR_ERR(other_crtc_state);
6506
6507                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6508                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6509                         return -EINVAL;
6510                 }
6511                 return 0;
6512         default:
6513                 BUG();
6514         }
6515 }
6516
6517 #define RETRY 1
6518 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6519                                        struct intel_crtc_state *pipe_config)
6520 {
6521         struct drm_device *dev = intel_crtc->base.dev;
6522         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6523         int lane, link_bw, fdi_dotclock, ret;
6524         bool needs_recompute = false;
6525
6526 retry:
6527         /* FDI is a binary signal running at ~2.7GHz, encoding
6528          * each output octet as 10 bits. The actual frequency
6529          * is stored as a divider into a 100MHz clock, and the
6530          * mode pixel clock is stored in units of 1KHz.
6531          * Hence the bw of each lane in terms of the mode signal
6532          * is:
6533          */
6534         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
6535
6536         fdi_dotclock = adjusted_mode->crtc_clock;
6537
6538         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6539                                            pipe_config->pipe_bpp);
6540
6541         pipe_config->fdi_lanes = lane;
6542
6543         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6544                                link_bw, &pipe_config->fdi_m_n, false);
6545
6546         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
6547         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6548                 pipe_config->pipe_bpp -= 2*3;
6549                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6550                               pipe_config->pipe_bpp);
6551                 needs_recompute = true;
6552                 pipe_config->bw_constrained = true;
6553
6554                 goto retry;
6555         }
6556
6557         if (needs_recompute)
6558                 return RETRY;
6559
6560         return ret;
6561 }
6562
6563 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
6564 {
6565         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6566         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6567
6568         /* IPS only exists on ULT machines and is tied to pipe A. */
6569         if (!hsw_crtc_supports_ips(crtc))
6570                 return false;
6571
6572         if (!i915_modparams.enable_ips)
6573                 return false;
6574
6575         if (crtc_state->pipe_bpp > 24)
6576                 return false;
6577
6578         /*
6579          * We compare against max which means we must take
6580          * the increased cdclk requirement into account when
6581          * calculating the new cdclk.
6582          *
6583          * Should measure whether using a lower cdclk w/o IPS
6584          */
6585         if (IS_BROADWELL(dev_priv) &&
6586             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
6587                 return false;
6588
6589         return true;
6590 }
6591
6592 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
6593 {
6594         struct drm_i915_private *dev_priv =
6595                 to_i915(crtc_state->base.crtc->dev);
6596         struct intel_atomic_state *intel_state =
6597                 to_intel_atomic_state(crtc_state->base.state);
6598
6599         if (!hsw_crtc_state_ips_capable(crtc_state))
6600                 return false;
6601
6602         if (crtc_state->ips_force_disable)
6603                 return false;
6604
6605         /* IPS should be fine as long as at least one plane is enabled. */
6606         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
6607                 return false;
6608
6609         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
6610         if (IS_BROADWELL(dev_priv) &&
6611             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
6612                 return false;
6613
6614         return true;
6615 }
6616
6617 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6618 {
6619         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6620
6621         /* GDG double wide on either pipe, otherwise pipe A only */
6622         return INTEL_GEN(dev_priv) < 4 &&
6623                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6624 }
6625
6626 static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
6627 {
6628         uint32_t pixel_rate;
6629
6630         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
6631
6632         /*
6633          * We only use IF-ID interlacing. If we ever use
6634          * PF-ID we'll need to adjust the pixel_rate here.
6635          */
6636
6637         if (pipe_config->pch_pfit.enabled) {
6638                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6639                 uint32_t pfit_size = pipe_config->pch_pfit.size;
6640
6641                 pipe_w = pipe_config->pipe_src_w;
6642                 pipe_h = pipe_config->pipe_src_h;
6643
6644                 pfit_w = (pfit_size >> 16) & 0xFFFF;
6645                 pfit_h = pfit_size & 0xFFFF;
6646                 if (pipe_w < pfit_w)
6647                         pipe_w = pfit_w;
6648                 if (pipe_h < pfit_h)
6649                         pipe_h = pfit_h;
6650
6651                 if (WARN_ON(!pfit_w || !pfit_h))
6652                         return pixel_rate;
6653
6654                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
6655                                      pfit_w * pfit_h);
6656         }
6657
6658         return pixel_rate;
6659 }
6660
6661 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
6662 {
6663         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
6664
6665         if (HAS_GMCH_DISPLAY(dev_priv))
6666                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
6667                 crtc_state->pixel_rate =
6668                         crtc_state->base.adjusted_mode.crtc_clock;
6669         else
6670                 crtc_state->pixel_rate =
6671                         ilk_pipe_pixel_rate(crtc_state);
6672 }
6673
6674 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6675                                      struct intel_crtc_state *pipe_config)
6676 {
6677         struct drm_device *dev = crtc->base.dev;
6678         struct drm_i915_private *dev_priv = to_i915(dev);
6679         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6680         int clock_limit = dev_priv->max_dotclk_freq;
6681
6682         if (INTEL_GEN(dev_priv) < 4) {
6683                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6684
6685                 /*
6686                  * Enable double wide mode when the dot clock
6687                  * is > 90% of the (display) core speed.
6688                  */
6689                 if (intel_crtc_supports_double_wide(crtc) &&
6690                     adjusted_mode->crtc_clock > clock_limit) {
6691                         clock_limit = dev_priv->max_dotclk_freq;
6692                         pipe_config->double_wide = true;
6693                 }
6694         }
6695
6696         if (adjusted_mode->crtc_clock > clock_limit) {
6697                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6698                               adjusted_mode->crtc_clock, clock_limit,
6699                               yesno(pipe_config->double_wide));
6700                 return -EINVAL;
6701         }
6702
6703         if (pipe_config->ycbcr420 && pipe_config->base.ctm) {
6704                 /*
6705                  * There is only one pipe CSC unit per pipe, and we need that
6706                  * for output conversion from RGB->YCBCR. So if CTM is already
6707                  * applied we can't support YCBCR420 output.
6708                  */
6709                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
6710                 return -EINVAL;
6711         }
6712
6713         /*
6714          * Pipe horizontal size must be even in:
6715          * - DVO ganged mode
6716          * - LVDS dual channel mode
6717          * - Double wide pipe
6718          */
6719         if (pipe_config->pipe_src_w & 1) {
6720                 if (pipe_config->double_wide) {
6721                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
6722                         return -EINVAL;
6723                 }
6724
6725                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6726                     intel_is_dual_link_lvds(dev)) {
6727                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
6728                         return -EINVAL;
6729                 }
6730         }
6731
6732         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6733          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6734          */
6735         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
6736                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6737                 return -EINVAL;
6738
6739         intel_crtc_compute_pixel_rate(pipe_config);
6740
6741         if (pipe_config->has_pch_encoder)
6742                 return ironlake_fdi_compute_config(crtc, pipe_config);
6743
6744         return 0;
6745 }
6746
6747 static void
6748 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
6749 {
6750         while (*num > DATA_LINK_M_N_MASK ||
6751                *den > DATA_LINK_M_N_MASK) {
6752                 *num >>= 1;
6753                 *den >>= 1;
6754         }
6755 }
6756
6757 static void compute_m_n(unsigned int m, unsigned int n,
6758                         uint32_t *ret_m, uint32_t *ret_n,
6759                         bool constant_n)
6760 {
6761         /*
6762          * Several DP dongles in particular seem to be fussy about
6763          * too large link M/N values. Give N value as 0x8000 that
6764          * should be acceptable by specific devices. 0x8000 is the
6765          * specified fixed N value for asynchronous clock mode,
6766          * which the devices expect also in synchronous clock mode.
6767          */
6768         if (constant_n)
6769                 *ret_n = 0x8000;
6770         else
6771                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6772
6773         *ret_m = div_u64((uint64_t) m * *ret_n, n);
6774         intel_reduce_m_n_ratio(ret_m, ret_n);
6775 }
6776
6777 void
6778 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
6779                        int pixel_clock, int link_clock,
6780                        struct intel_link_m_n *m_n,
6781                        bool constant_n)
6782 {
6783         m_n->tu = 64;
6784
6785         compute_m_n(bits_per_pixel * pixel_clock,
6786                     link_clock * nlanes * 8,
6787                     &m_n->gmch_m, &m_n->gmch_n,
6788                     constant_n);
6789
6790         compute_m_n(pixel_clock, link_clock,
6791                     &m_n->link_m, &m_n->link_n,
6792                     constant_n);
6793 }
6794
6795 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
6796 {
6797         if (i915_modparams.panel_use_ssc >= 0)
6798                 return i915_modparams.panel_use_ssc != 0;
6799         return dev_priv->vbt.lvds_use_ssc
6800                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
6801 }
6802
6803 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
6804 {
6805         return (1 << dpll->n) << 16 | dpll->m2;
6806 }
6807
6808 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
6809 {
6810         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
6811 }
6812
6813 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
6814                                      struct intel_crtc_state *crtc_state,
6815                                      struct dpll *reduced_clock)
6816 {
6817         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6818         u32 fp, fp2 = 0;
6819
6820         if (IS_PINEVIEW(dev_priv)) {
6821                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
6822                 if (reduced_clock)
6823                         fp2 = pnv_dpll_compute_fp(reduced_clock);
6824         } else {
6825                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
6826                 if (reduced_clock)
6827                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
6828         }
6829
6830         crtc_state->dpll_hw_state.fp0 = fp;
6831
6832         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
6833             reduced_clock) {
6834                 crtc_state->dpll_hw_state.fp1 = fp2;
6835         } else {
6836                 crtc_state->dpll_hw_state.fp1 = fp;
6837         }
6838 }
6839
6840 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
6841                 pipe)
6842 {
6843         u32 reg_val;
6844
6845         /*
6846          * PLLB opamp always calibrates to max value of 0x3f, force enable it
6847          * and set it to a reasonable value instead.
6848          */
6849         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6850         reg_val &= 0xffffff00;
6851         reg_val |= 0x00000030;
6852         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6853
6854         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6855         reg_val &= 0x00ffffff;
6856         reg_val |= 0x8c000000;
6857         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6858
6859         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6860         reg_val &= 0xffffff00;
6861         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6862
6863         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6864         reg_val &= 0x00ffffff;
6865         reg_val |= 0xb0000000;
6866         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6867 }
6868
6869 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
6870                                          struct intel_link_m_n *m_n)
6871 {
6872         struct drm_device *dev = crtc->base.dev;
6873         struct drm_i915_private *dev_priv = to_i915(dev);
6874         int pipe = crtc->pipe;
6875
6876         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6877         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
6878         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
6879         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
6880 }
6881
6882 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
6883                                          struct intel_link_m_n *m_n,
6884                                          struct intel_link_m_n *m2_n2)
6885 {
6886         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6887         int pipe = crtc->pipe;
6888         enum transcoder transcoder = crtc->config->cpu_transcoder;
6889
6890         if (INTEL_GEN(dev_priv) >= 5) {
6891                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
6892                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
6893                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
6894                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
6895                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
6896                  * for gen < 8) and if DRRS is supported (to make sure the
6897                  * registers are not unnecessarily accessed).
6898                  */
6899                 if (m2_n2 && (IS_CHERRYVIEW(dev_priv) ||
6900                     INTEL_GEN(dev_priv) < 8) && crtc->config->has_drrs) {
6901                         I915_WRITE(PIPE_DATA_M2(transcoder),
6902                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
6903                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
6904                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
6905                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
6906                 }
6907         } else {
6908                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6909                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
6910                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
6911                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
6912         }
6913 }
6914
6915 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
6916 {
6917         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
6918
6919         if (m_n == M1_N1) {
6920                 dp_m_n = &crtc->config->dp_m_n;
6921                 dp_m2_n2 = &crtc->config->dp_m2_n2;
6922         } else if (m_n == M2_N2) {
6923
6924                 /*
6925                  * M2_N2 registers are not supported. Hence m2_n2 divider value
6926                  * needs to be programmed into M1_N1.
6927                  */
6928                 dp_m_n = &crtc->config->dp_m2_n2;
6929         } else {
6930                 DRM_ERROR("Unsupported divider value\n");
6931                 return;
6932         }
6933
6934         if (crtc->config->has_pch_encoder)
6935                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
6936         else
6937                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
6938 }
6939
6940 static void vlv_compute_dpll(struct intel_crtc *crtc,
6941                              struct intel_crtc_state *pipe_config)
6942 {
6943         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
6944                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6945         if (crtc->pipe != PIPE_A)
6946                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6947
6948         /* DPLL not used with DSI, but still need the rest set up */
6949         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6950                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
6951                         DPLL_EXT_BUFFER_ENABLE_VLV;
6952
6953         pipe_config->dpll_hw_state.dpll_md =
6954                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6955 }
6956
6957 static void chv_compute_dpll(struct intel_crtc *crtc,
6958                              struct intel_crtc_state *pipe_config)
6959 {
6960         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
6961                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6962         if (crtc->pipe != PIPE_A)
6963                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6964
6965         /* DPLL not used with DSI, but still need the rest set up */
6966         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6967                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
6968
6969         pipe_config->dpll_hw_state.dpll_md =
6970                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6971 }
6972
6973 static void vlv_prepare_pll(struct intel_crtc *crtc,
6974                             const struct intel_crtc_state *pipe_config)
6975 {
6976         struct drm_device *dev = crtc->base.dev;
6977         struct drm_i915_private *dev_priv = to_i915(dev);
6978         enum pipe pipe = crtc->pipe;
6979         u32 mdiv;
6980         u32 bestn, bestm1, bestm2, bestp1, bestp2;
6981         u32 coreclk, reg_val;
6982
6983         /* Enable Refclk */
6984         I915_WRITE(DPLL(pipe),
6985                    pipe_config->dpll_hw_state.dpll &
6986                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
6987
6988         /* No need to actually set up the DPLL with DSI */
6989         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
6990                 return;
6991
6992         mutex_lock(&dev_priv->sb_lock);
6993
6994         bestn = pipe_config->dpll.n;
6995         bestm1 = pipe_config->dpll.m1;
6996         bestm2 = pipe_config->dpll.m2;
6997         bestp1 = pipe_config->dpll.p1;
6998         bestp2 = pipe_config->dpll.p2;
6999
7000         /* See eDP HDMI DPIO driver vbios notes doc */
7001
7002         /* PLL B needs special handling */
7003         if (pipe == PIPE_B)
7004                 vlv_pllb_recal_opamp(dev_priv, pipe);
7005
7006         /* Set up Tx target for periodic Rcomp update */
7007         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7008
7009         /* Disable target IRef on PLL */
7010         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7011         reg_val &= 0x00ffffff;
7012         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7013
7014         /* Disable fast lock */
7015         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7016
7017         /* Set idtafcrecal before PLL is enabled */
7018         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7019         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7020         mdiv |= ((bestn << DPIO_N_SHIFT));
7021         mdiv |= (1 << DPIO_K_SHIFT);
7022
7023         /*
7024          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7025          * but we don't support that).
7026          * Note: don't use the DAC post divider as it seems unstable.
7027          */
7028         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7029         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7030
7031         mdiv |= DPIO_ENABLE_CALIBRATION;
7032         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7033
7034         /* Set HBR and RBR LPF coefficients */
7035         if (pipe_config->port_clock == 162000 ||
7036             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_ANALOG) ||
7037             intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
7038                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7039                                  0x009f0003);
7040         else
7041                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7042                                  0x00d0000f);
7043
7044         if (intel_crtc_has_dp_encoder(pipe_config)) {
7045                 /* Use SSC source */
7046                 if (pipe == PIPE_A)
7047                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7048                                          0x0df40000);
7049                 else
7050                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7051                                          0x0df70000);
7052         } else { /* HDMI or VGA */
7053                 /* Use bend source */
7054                 if (pipe == PIPE_A)
7055                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7056                                          0x0df70000);
7057                 else
7058                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7059                                          0x0df40000);
7060         }
7061
7062         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7063         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7064         if (intel_crtc_has_dp_encoder(crtc->config))
7065                 coreclk |= 0x01000000;
7066         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7067
7068         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7069         mutex_unlock(&dev_priv->sb_lock);
7070 }
7071
7072 static void chv_prepare_pll(struct intel_crtc *crtc,
7073                             const struct intel_crtc_state *pipe_config)
7074 {
7075         struct drm_device *dev = crtc->base.dev;
7076         struct drm_i915_private *dev_priv = to_i915(dev);
7077         enum pipe pipe = crtc->pipe;
7078         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7079         u32 loopfilter, tribuf_calcntr;
7080         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7081         u32 dpio_val;
7082         int vco;
7083
7084         /* Enable Refclk and SSC */
7085         I915_WRITE(DPLL(pipe),
7086                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7087
7088         /* No need to actually set up the DPLL with DSI */
7089         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7090                 return;
7091
7092         bestn = pipe_config->dpll.n;
7093         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7094         bestm1 = pipe_config->dpll.m1;
7095         bestm2 = pipe_config->dpll.m2 >> 22;
7096         bestp1 = pipe_config->dpll.p1;
7097         bestp2 = pipe_config->dpll.p2;
7098         vco = pipe_config->dpll.vco;
7099         dpio_val = 0;
7100         loopfilter = 0;
7101
7102         mutex_lock(&dev_priv->sb_lock);
7103
7104         /* p1 and p2 divider */
7105         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7106                         5 << DPIO_CHV_S1_DIV_SHIFT |
7107                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7108                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7109                         1 << DPIO_CHV_K_DIV_SHIFT);
7110
7111         /* Feedback post-divider - m2 */
7112         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7113
7114         /* Feedback refclk divider - n and m1 */
7115         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7116                         DPIO_CHV_M1_DIV_BY_2 |
7117                         1 << DPIO_CHV_N_DIV_SHIFT);
7118
7119         /* M2 fraction division */
7120         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7121
7122         /* M2 fraction division enable */
7123         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7124         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7125         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7126         if (bestm2_frac)
7127                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7128         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7129
7130         /* Program digital lock detect threshold */
7131         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7132         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7133                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7134         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7135         if (!bestm2_frac)
7136                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7137         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7138
7139         /* Loop filter */
7140         if (vco == 5400000) {
7141                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7142                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7143                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7144                 tribuf_calcntr = 0x9;
7145         } else if (vco <= 6200000) {
7146                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7147                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7148                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7149                 tribuf_calcntr = 0x9;
7150         } else if (vco <= 6480000) {
7151                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7152                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7153                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7154                 tribuf_calcntr = 0x8;
7155         } else {
7156                 /* Not supported. Apply the same limits as in the max case */
7157                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7158                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7159                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7160                 tribuf_calcntr = 0;
7161         }
7162         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7163
7164         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7165         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7166         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7167         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7168
7169         /* AFC Recal */
7170         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7171                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7172                         DPIO_AFC_RECAL);
7173
7174         mutex_unlock(&dev_priv->sb_lock);
7175 }
7176
7177 /**
7178  * vlv_force_pll_on - forcibly enable just the PLL
7179  * @dev_priv: i915 private structure
7180  * @pipe: pipe PLL to enable
7181  * @dpll: PLL configuration
7182  *
7183  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7184  * in cases where we need the PLL enabled even when @pipe is not going to
7185  * be enabled.
7186  */
7187 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7188                      const struct dpll *dpll)
7189 {
7190         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7191         struct intel_crtc_state *pipe_config;
7192
7193         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7194         if (!pipe_config)
7195                 return -ENOMEM;
7196
7197         pipe_config->base.crtc = &crtc->base;
7198         pipe_config->pixel_multiplier = 1;
7199         pipe_config->dpll = *dpll;
7200
7201         if (IS_CHERRYVIEW(dev_priv)) {
7202                 chv_compute_dpll(crtc, pipe_config);
7203                 chv_prepare_pll(crtc, pipe_config);
7204                 chv_enable_pll(crtc, pipe_config);
7205         } else {
7206                 vlv_compute_dpll(crtc, pipe_config);
7207                 vlv_prepare_pll(crtc, pipe_config);
7208                 vlv_enable_pll(crtc, pipe_config);
7209         }
7210
7211         kfree(pipe_config);
7212
7213         return 0;
7214 }
7215
7216 /**
7217  * vlv_force_pll_off - forcibly disable just the PLL
7218  * @dev_priv: i915 private structure
7219  * @pipe: pipe PLL to disable
7220  *
7221  * Disable the PLL for @pipe. To be used in cases where we need
7222  * the PLL enabled even when @pipe is not going to be enabled.
7223  */
7224 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7225 {
7226         if (IS_CHERRYVIEW(dev_priv))
7227                 chv_disable_pll(dev_priv, pipe);
7228         else
7229                 vlv_disable_pll(dev_priv, pipe);
7230 }
7231
7232 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7233                               struct intel_crtc_state *crtc_state,
7234                               struct dpll *reduced_clock)
7235 {
7236         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7237         u32 dpll;
7238         struct dpll *clock = &crtc_state->dpll;
7239
7240         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7241
7242         dpll = DPLL_VGA_MODE_DIS;
7243
7244         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7245                 dpll |= DPLLB_MODE_LVDS;
7246         else
7247                 dpll |= DPLLB_MODE_DAC_SERIAL;
7248
7249         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7250             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7251                 dpll |= (crtc_state->pixel_multiplier - 1)
7252                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7253         }
7254
7255         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7256             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
7257                 dpll |= DPLL_SDVO_HIGH_SPEED;
7258
7259         if (intel_crtc_has_dp_encoder(crtc_state))
7260                 dpll |= DPLL_SDVO_HIGH_SPEED;
7261
7262         /* compute bitmask from p1 value */
7263         if (IS_PINEVIEW(dev_priv))
7264                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7265         else {
7266                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7267                 if (IS_G4X(dev_priv) && reduced_clock)
7268                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7269         }
7270         switch (clock->p2) {
7271         case 5:
7272                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7273                 break;
7274         case 7:
7275                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7276                 break;
7277         case 10:
7278                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7279                 break;
7280         case 14:
7281                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7282                 break;
7283         }
7284         if (INTEL_GEN(dev_priv) >= 4)
7285                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7286
7287         if (crtc_state->sdvo_tv_clock)
7288                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7289         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7290                  intel_panel_use_ssc(dev_priv))
7291                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7292         else
7293                 dpll |= PLL_REF_INPUT_DREFCLK;
7294
7295         dpll |= DPLL_VCO_ENABLE;
7296         crtc_state->dpll_hw_state.dpll = dpll;
7297
7298         if (INTEL_GEN(dev_priv) >= 4) {
7299                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7300                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7301                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7302         }
7303 }
7304
7305 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7306                               struct intel_crtc_state *crtc_state,
7307                               struct dpll *reduced_clock)
7308 {
7309         struct drm_device *dev = crtc->base.dev;
7310         struct drm_i915_private *dev_priv = to_i915(dev);
7311         u32 dpll;
7312         struct dpll *clock = &crtc_state->dpll;
7313
7314         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7315
7316         dpll = DPLL_VGA_MODE_DIS;
7317
7318         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7319                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7320         } else {
7321                 if (clock->p1 == 2)
7322                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7323                 else
7324                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7325                 if (clock->p2 == 4)
7326                         dpll |= PLL_P2_DIVIDE_BY_4;
7327         }
7328
7329         if (!IS_I830(dev_priv) &&
7330             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
7331                 dpll |= DPLL_DVO_2X_MODE;
7332
7333         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7334             intel_panel_use_ssc(dev_priv))
7335                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7336         else
7337                 dpll |= PLL_REF_INPUT_DREFCLK;
7338
7339         dpll |= DPLL_VCO_ENABLE;
7340         crtc_state->dpll_hw_state.dpll = dpll;
7341 }
7342
7343 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7344 {
7345         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
7346         enum pipe pipe = intel_crtc->pipe;
7347         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7348         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
7349         uint32_t crtc_vtotal, crtc_vblank_end;
7350         int vsyncshift = 0;
7351
7352         /* We need to be careful not to changed the adjusted mode, for otherwise
7353          * the hw state checker will get angry at the mismatch. */
7354         crtc_vtotal = adjusted_mode->crtc_vtotal;
7355         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7356
7357         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7358                 /* the chip adds 2 halflines automatically */
7359                 crtc_vtotal -= 1;
7360                 crtc_vblank_end -= 1;
7361
7362                 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
7363                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7364                 else
7365                         vsyncshift = adjusted_mode->crtc_hsync_start -
7366                                 adjusted_mode->crtc_htotal / 2;
7367                 if (vsyncshift < 0)
7368                         vsyncshift += adjusted_mode->crtc_htotal;
7369         }
7370
7371         if (INTEL_GEN(dev_priv) > 3)
7372                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7373
7374         I915_WRITE(HTOTAL(cpu_transcoder),
7375                    (adjusted_mode->crtc_hdisplay - 1) |
7376                    ((adjusted_mode->crtc_htotal - 1) << 16));
7377         I915_WRITE(HBLANK(cpu_transcoder),
7378                    (adjusted_mode->crtc_hblank_start - 1) |
7379                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7380         I915_WRITE(HSYNC(cpu_transcoder),
7381                    (adjusted_mode->crtc_hsync_start - 1) |
7382                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7383
7384         I915_WRITE(VTOTAL(cpu_transcoder),
7385                    (adjusted_mode->crtc_vdisplay - 1) |
7386                    ((crtc_vtotal - 1) << 16));
7387         I915_WRITE(VBLANK(cpu_transcoder),
7388                    (adjusted_mode->crtc_vblank_start - 1) |
7389                    ((crtc_vblank_end - 1) << 16));
7390         I915_WRITE(VSYNC(cpu_transcoder),
7391                    (adjusted_mode->crtc_vsync_start - 1) |
7392                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7393
7394         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7395          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7396          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7397          * bits. */
7398         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
7399             (pipe == PIPE_B || pipe == PIPE_C))
7400                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7401
7402 }
7403
7404 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc)
7405 {
7406         struct drm_device *dev = intel_crtc->base.dev;
7407         struct drm_i915_private *dev_priv = to_i915(dev);
7408         enum pipe pipe = intel_crtc->pipe;
7409
7410         /* pipesrc controls the size that is scaled from, which should
7411          * always be the user's requested size.
7412          */
7413         I915_WRITE(PIPESRC(pipe),
7414                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7415                    (intel_crtc->config->pipe_src_h - 1));
7416 }
7417
7418 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7419                                    struct intel_crtc_state *pipe_config)
7420 {
7421         struct drm_device *dev = crtc->base.dev;
7422         struct drm_i915_private *dev_priv = to_i915(dev);
7423         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7424         uint32_t tmp;
7425
7426         tmp = I915_READ(HTOTAL(cpu_transcoder));
7427         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7428         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7429         tmp = I915_READ(HBLANK(cpu_transcoder));
7430         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7431         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7432         tmp = I915_READ(HSYNC(cpu_transcoder));
7433         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7434         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7435
7436         tmp = I915_READ(VTOTAL(cpu_transcoder));
7437         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7438         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7439         tmp = I915_READ(VBLANK(cpu_transcoder));
7440         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7441         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7442         tmp = I915_READ(VSYNC(cpu_transcoder));
7443         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7444         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7445
7446         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7447                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7448                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7449                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7450         }
7451 }
7452
7453 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7454                                     struct intel_crtc_state *pipe_config)
7455 {
7456         struct drm_device *dev = crtc->base.dev;
7457         struct drm_i915_private *dev_priv = to_i915(dev);
7458         u32 tmp;
7459
7460         tmp = I915_READ(PIPESRC(crtc->pipe));
7461         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7462         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7463
7464         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7465         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7466 }
7467
7468 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7469                                  struct intel_crtc_state *pipe_config)
7470 {
7471         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7472         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7473         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7474         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7475
7476         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7477         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7478         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7479         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7480
7481         mode->flags = pipe_config->base.adjusted_mode.flags;
7482         mode->type = DRM_MODE_TYPE_DRIVER;
7483
7484         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7485
7486         mode->hsync = drm_mode_hsync(mode);
7487         mode->vrefresh = drm_mode_vrefresh(mode);
7488         drm_mode_set_name(mode);
7489 }
7490
7491 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
7492 {
7493         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7494         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7495         uint32_t pipeconf;
7496
7497         pipeconf = 0;
7498
7499         /* we keep both pipes enabled on 830 */
7500         if (IS_I830(dev_priv))
7501                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
7502
7503         if (crtc_state->double_wide)
7504                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7505
7506         /* only g4x and later have fancy bpc/dither controls */
7507         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7508             IS_CHERRYVIEW(dev_priv)) {
7509                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7510                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
7511                         pipeconf |= PIPECONF_DITHER_EN |
7512                                     PIPECONF_DITHER_TYPE_SP;
7513
7514                 switch (crtc_state->pipe_bpp) {
7515                 case 18:
7516                         pipeconf |= PIPECONF_6BPC;
7517                         break;
7518                 case 24:
7519                         pipeconf |= PIPECONF_8BPC;
7520                         break;
7521                 case 30:
7522                         pipeconf |= PIPECONF_10BPC;
7523                         break;
7524                 default:
7525                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7526                         BUG();
7527                 }
7528         }
7529
7530         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7531                 if (INTEL_GEN(dev_priv) < 4 ||
7532                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7533                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7534                 else
7535                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7536         } else
7537                 pipeconf |= PIPECONF_PROGRESSIVE;
7538
7539         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7540              crtc_state->limited_color_range)
7541                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7542
7543         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
7544         POSTING_READ(PIPECONF(crtc->pipe));
7545 }
7546
7547 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
7548                                    struct intel_crtc_state *crtc_state)
7549 {
7550         struct drm_device *dev = crtc->base.dev;
7551         struct drm_i915_private *dev_priv = to_i915(dev);
7552         const struct intel_limit *limit;
7553         int refclk = 48000;
7554
7555         memset(&crtc_state->dpll_hw_state, 0,
7556                sizeof(crtc_state->dpll_hw_state));
7557
7558         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7559                 if (intel_panel_use_ssc(dev_priv)) {
7560                         refclk = dev_priv->vbt.lvds_ssc_freq;
7561                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7562                 }
7563
7564                 limit = &intel_limits_i8xx_lvds;
7565         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
7566                 limit = &intel_limits_i8xx_dvo;
7567         } else {
7568                 limit = &intel_limits_i8xx_dac;
7569         }
7570
7571         if (!crtc_state->clock_set &&
7572             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7573                                  refclk, NULL, &crtc_state->dpll)) {
7574                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7575                 return -EINVAL;
7576         }
7577
7578         i8xx_compute_dpll(crtc, crtc_state, NULL);
7579
7580         return 0;
7581 }
7582
7583 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
7584                                   struct intel_crtc_state *crtc_state)
7585 {
7586         struct drm_device *dev = crtc->base.dev;
7587         struct drm_i915_private *dev_priv = to_i915(dev);
7588         const struct intel_limit *limit;
7589         int refclk = 96000;
7590
7591         memset(&crtc_state->dpll_hw_state, 0,
7592                sizeof(crtc_state->dpll_hw_state));
7593
7594         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7595                 if (intel_panel_use_ssc(dev_priv)) {
7596                         refclk = dev_priv->vbt.lvds_ssc_freq;
7597                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7598                 }
7599
7600                 if (intel_is_dual_link_lvds(dev))
7601                         limit = &intel_limits_g4x_dual_channel_lvds;
7602                 else
7603                         limit = &intel_limits_g4x_single_channel_lvds;
7604         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
7605                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
7606                 limit = &intel_limits_g4x_hdmi;
7607         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
7608                 limit = &intel_limits_g4x_sdvo;
7609         } else {
7610                 /* The option is for other outputs */
7611                 limit = &intel_limits_i9xx_sdvo;
7612         }
7613
7614         if (!crtc_state->clock_set &&
7615             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7616                                 refclk, NULL, &crtc_state->dpll)) {
7617                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7618                 return -EINVAL;
7619         }
7620
7621         i9xx_compute_dpll(crtc, crtc_state, NULL);
7622
7623         return 0;
7624 }
7625
7626 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
7627                                   struct intel_crtc_state *crtc_state)
7628 {
7629         struct drm_device *dev = crtc->base.dev;
7630         struct drm_i915_private *dev_priv = to_i915(dev);
7631         const struct intel_limit *limit;
7632         int refclk = 96000;
7633
7634         memset(&crtc_state->dpll_hw_state, 0,
7635                sizeof(crtc_state->dpll_hw_state));
7636
7637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7638                 if (intel_panel_use_ssc(dev_priv)) {
7639                         refclk = dev_priv->vbt.lvds_ssc_freq;
7640                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7641                 }
7642
7643                 limit = &intel_limits_pineview_lvds;
7644         } else {
7645                 limit = &intel_limits_pineview_sdvo;
7646         }
7647
7648         if (!crtc_state->clock_set &&
7649             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7650                                 refclk, NULL, &crtc_state->dpll)) {
7651                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7652                 return -EINVAL;
7653         }
7654
7655         i9xx_compute_dpll(crtc, crtc_state, NULL);
7656
7657         return 0;
7658 }
7659
7660 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7661                                    struct intel_crtc_state *crtc_state)
7662 {
7663         struct drm_device *dev = crtc->base.dev;
7664         struct drm_i915_private *dev_priv = to_i915(dev);
7665         const struct intel_limit *limit;
7666         int refclk = 96000;
7667
7668         memset(&crtc_state->dpll_hw_state, 0,
7669                sizeof(crtc_state->dpll_hw_state));
7670
7671         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7672                 if (intel_panel_use_ssc(dev_priv)) {
7673                         refclk = dev_priv->vbt.lvds_ssc_freq;
7674                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7675                 }
7676
7677                 limit = &intel_limits_i9xx_lvds;
7678         } else {
7679                 limit = &intel_limits_i9xx_sdvo;
7680         }
7681
7682         if (!crtc_state->clock_set &&
7683             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7684                                  refclk, NULL, &crtc_state->dpll)) {
7685                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7686                 return -EINVAL;
7687         }
7688
7689         i9xx_compute_dpll(crtc, crtc_state, NULL);
7690
7691         return 0;
7692 }
7693
7694 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
7695                                   struct intel_crtc_state *crtc_state)
7696 {
7697         int refclk = 100000;
7698         const struct intel_limit *limit = &intel_limits_chv;
7699
7700         memset(&crtc_state->dpll_hw_state, 0,
7701                sizeof(crtc_state->dpll_hw_state));
7702
7703         if (!crtc_state->clock_set &&
7704             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7705                                 refclk, NULL, &crtc_state->dpll)) {
7706                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7707                 return -EINVAL;
7708         }
7709
7710         chv_compute_dpll(crtc, crtc_state);
7711
7712         return 0;
7713 }
7714
7715 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
7716                                   struct intel_crtc_state *crtc_state)
7717 {
7718         int refclk = 100000;
7719         const struct intel_limit *limit = &intel_limits_vlv;
7720
7721         memset(&crtc_state->dpll_hw_state, 0,
7722                sizeof(crtc_state->dpll_hw_state));
7723
7724         if (!crtc_state->clock_set &&
7725             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7726                                 refclk, NULL, &crtc_state->dpll)) {
7727                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7728                 return -EINVAL;
7729         }
7730
7731         vlv_compute_dpll(crtc, crtc_state);
7732
7733         return 0;
7734 }
7735
7736 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7737                                  struct intel_crtc_state *pipe_config)
7738 {
7739         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7740         uint32_t tmp;
7741
7742         if (INTEL_GEN(dev_priv) <= 3 &&
7743             (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
7744                 return;
7745
7746         tmp = I915_READ(PFIT_CONTROL);
7747         if (!(tmp & PFIT_ENABLE))
7748                 return;
7749
7750         /* Check whether the pfit is attached to our pipe. */
7751         if (INTEL_GEN(dev_priv) < 4) {
7752                 if (crtc->pipe != PIPE_B)
7753                         return;
7754         } else {
7755                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7756                         return;
7757         }
7758
7759         pipe_config->gmch_pfit.control = tmp;
7760         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7761 }
7762
7763 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7764                                struct intel_crtc_state *pipe_config)
7765 {
7766         struct drm_device *dev = crtc->base.dev;
7767         struct drm_i915_private *dev_priv = to_i915(dev);
7768         int pipe = pipe_config->cpu_transcoder;
7769         struct dpll clock;
7770         u32 mdiv;
7771         int refclk = 100000;
7772
7773         /* In case of DSI, DPLL will not be used */
7774         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7775                 return;
7776
7777         mutex_lock(&dev_priv->sb_lock);
7778         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7779         mutex_unlock(&dev_priv->sb_lock);
7780
7781         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7782         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7783         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7784         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7785         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7786
7787         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
7788 }
7789
7790 static void
7791 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7792                               struct intel_initial_plane_config *plane_config)
7793 {
7794         struct drm_device *dev = crtc->base.dev;
7795         struct drm_i915_private *dev_priv = to_i915(dev);
7796         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7797         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7798         enum pipe pipe;
7799         u32 val, base, offset;
7800         int fourcc, pixel_format;
7801         unsigned int aligned_height;
7802         struct drm_framebuffer *fb;
7803         struct intel_framebuffer *intel_fb;
7804
7805         if (!plane->get_hw_state(plane, &pipe))
7806                 return;
7807
7808         WARN_ON(pipe != crtc->pipe);
7809
7810         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7811         if (!intel_fb) {
7812                 DRM_DEBUG_KMS("failed to alloc fb\n");
7813                 return;
7814         }
7815
7816         fb = &intel_fb->base;
7817
7818         fb->dev = dev;
7819
7820         val = I915_READ(DSPCNTR(i9xx_plane));
7821
7822         if (INTEL_GEN(dev_priv) >= 4) {
7823                 if (val & DISPPLANE_TILED) {
7824                         plane_config->tiling = I915_TILING_X;
7825                         fb->modifier = I915_FORMAT_MOD_X_TILED;
7826                 }
7827         }
7828
7829         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7830         fourcc = i9xx_format_to_fourcc(pixel_format);
7831         fb->format = drm_format_info(fourcc);
7832
7833         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7834                 offset = I915_READ(DSPOFFSET(i9xx_plane));
7835                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7836         } else if (INTEL_GEN(dev_priv) >= 4) {
7837                 if (plane_config->tiling)
7838                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
7839                 else
7840                         offset = I915_READ(DSPLINOFF(i9xx_plane));
7841                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7842         } else {
7843                 base = I915_READ(DSPADDR(i9xx_plane));
7844         }
7845         plane_config->base = base;
7846
7847         val = I915_READ(PIPESRC(pipe));
7848         fb->width = ((val >> 16) & 0xfff) + 1;
7849         fb->height = ((val >> 0) & 0xfff) + 1;
7850
7851         val = I915_READ(DSPSTRIDE(i9xx_plane));
7852         fb->pitches[0] = val & 0xffffffc0;
7853
7854         aligned_height = intel_fb_align_height(fb, 0, fb->height);
7855
7856         plane_config->size = fb->pitches[0] * aligned_height;
7857
7858         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7859                       crtc->base.name, plane->base.name, fb->width, fb->height,
7860                       fb->format->cpp[0] * 8, base, fb->pitches[0],
7861                       plane_config->size);
7862
7863         plane_config->fb = intel_fb;
7864 }
7865
7866 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7867                                struct intel_crtc_state *pipe_config)
7868 {
7869         struct drm_device *dev = crtc->base.dev;
7870         struct drm_i915_private *dev_priv = to_i915(dev);
7871         int pipe = pipe_config->cpu_transcoder;
7872         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7873         struct dpll clock;
7874         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
7875         int refclk = 100000;
7876
7877         /* In case of DSI, DPLL will not be used */
7878         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7879                 return;
7880
7881         mutex_lock(&dev_priv->sb_lock);
7882         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7883         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7884         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7885         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7886         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7887         mutex_unlock(&dev_priv->sb_lock);
7888
7889         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7890         clock.m2 = (pll_dw0 & 0xff) << 22;
7891         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
7892                 clock.m2 |= pll_dw2 & 0x3fffff;
7893         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7894         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7895         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7896
7897         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
7898 }
7899
7900 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7901                                  struct intel_crtc_state *pipe_config)
7902 {
7903         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7904         enum intel_display_power_domain power_domain;
7905         uint32_t tmp;
7906         bool ret;
7907
7908         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
7909         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
7910                 return false;
7911
7912         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
7913         pipe_config->shared_dpll = NULL;
7914
7915         ret = false;
7916
7917         tmp = I915_READ(PIPECONF(crtc->pipe));
7918         if (!(tmp & PIPECONF_ENABLE))
7919                 goto out;
7920
7921         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7922             IS_CHERRYVIEW(dev_priv)) {
7923                 switch (tmp & PIPECONF_BPC_MASK) {
7924                 case PIPECONF_6BPC:
7925                         pipe_config->pipe_bpp = 18;
7926                         break;
7927                 case PIPECONF_8BPC:
7928                         pipe_config->pipe_bpp = 24;
7929                         break;
7930                 case PIPECONF_10BPC:
7931                         pipe_config->pipe_bpp = 30;
7932                         break;
7933                 default:
7934                         break;
7935                 }
7936         }
7937
7938         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7939             (tmp & PIPECONF_COLOR_RANGE_SELECT))
7940                 pipe_config->limited_color_range = true;
7941
7942         if (INTEL_GEN(dev_priv) < 4)
7943                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
7944
7945         intel_get_pipe_timings(crtc, pipe_config);
7946         intel_get_pipe_src_size(crtc, pipe_config);
7947
7948         i9xx_get_pfit_config(crtc, pipe_config);
7949
7950         if (INTEL_GEN(dev_priv) >= 4) {
7951                 /* No way to read it out on pipes B and C */
7952                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
7953                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
7954                 else
7955                         tmp = I915_READ(DPLL_MD(crtc->pipe));
7956                 pipe_config->pixel_multiplier =
7957                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
7958                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
7959                 pipe_config->dpll_hw_state.dpll_md = tmp;
7960         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7961                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7962                 tmp = I915_READ(DPLL(crtc->pipe));
7963                 pipe_config->pixel_multiplier =
7964                         ((tmp & SDVO_MULTIPLIER_MASK)
7965                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
7966         } else {
7967                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
7968                  * port and will be fixed up in the encoder->get_config
7969                  * function. */
7970                 pipe_config->pixel_multiplier = 1;
7971         }
7972         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
7973         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
7974                 /*
7975                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
7976                  * on 830. Filter it out here so that we don't
7977                  * report errors due to that.
7978                  */
7979                 if (IS_I830(dev_priv))
7980                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
7981
7982                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
7983                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
7984         } else {
7985                 /* Mask out read-only status bits. */
7986                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
7987                                                      DPLL_PORTC_READY_MASK |
7988                                                      DPLL_PORTB_READY_MASK);
7989         }
7990
7991         if (IS_CHERRYVIEW(dev_priv))
7992                 chv_crtc_clock_get(crtc, pipe_config);
7993         else if (IS_VALLEYVIEW(dev_priv))
7994                 vlv_crtc_clock_get(crtc, pipe_config);
7995         else
7996                 i9xx_crtc_clock_get(crtc, pipe_config);
7997
7998         /*
7999          * Normally the dotclock is filled in by the encoder .get_config()
8000          * but in case the pipe is enabled w/o any ports we need a sane
8001          * default.
8002          */
8003         pipe_config->base.adjusted_mode.crtc_clock =
8004                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8005
8006         ret = true;
8007
8008 out:
8009         intel_display_power_put(dev_priv, power_domain);
8010
8011         return ret;
8012 }
8013
8014 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8015 {
8016         struct intel_encoder *encoder;
8017         int i;
8018         u32 val, final;
8019         bool has_lvds = false;
8020         bool has_cpu_edp = false;
8021         bool has_panel = false;
8022         bool has_ck505 = false;
8023         bool can_ssc = false;
8024         bool using_ssc_source = false;
8025
8026         /* We need to take the global config into account */
8027         for_each_intel_encoder(&dev_priv->drm, encoder) {
8028                 switch (encoder->type) {
8029                 case INTEL_OUTPUT_LVDS:
8030                         has_panel = true;
8031                         has_lvds = true;
8032                         break;
8033                 case INTEL_OUTPUT_EDP:
8034                         has_panel = true;
8035                         if (encoder->port == PORT_A)
8036                                 has_cpu_edp = true;
8037                         break;
8038                 default:
8039                         break;
8040                 }
8041         }
8042
8043         if (HAS_PCH_IBX(dev_priv)) {
8044                 has_ck505 = dev_priv->vbt.display_clock_mode;
8045                 can_ssc = has_ck505;
8046         } else {
8047                 has_ck505 = false;
8048                 can_ssc = true;
8049         }
8050
8051         /* Check if any DPLLs are using the SSC source */
8052         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8053                 u32 temp = I915_READ(PCH_DPLL(i));
8054
8055                 if (!(temp & DPLL_VCO_ENABLE))
8056                         continue;
8057
8058                 if ((temp & PLL_REF_INPUT_MASK) ==
8059                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8060                         using_ssc_source = true;
8061                         break;
8062                 }
8063         }
8064
8065         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8066                       has_panel, has_lvds, has_ck505, using_ssc_source);
8067
8068         /* Ironlake: try to setup display ref clock before DPLL
8069          * enabling. This is only under driver's control after
8070          * PCH B stepping, previous chipset stepping should be
8071          * ignoring this setting.
8072          */
8073         val = I915_READ(PCH_DREF_CONTROL);
8074
8075         /* As we must carefully and slowly disable/enable each source in turn,
8076          * compute the final state we want first and check if we need to
8077          * make any changes at all.
8078          */
8079         final = val;
8080         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8081         if (has_ck505)
8082                 final |= DREF_NONSPREAD_CK505_ENABLE;
8083         else
8084                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8085
8086         final &= ~DREF_SSC_SOURCE_MASK;
8087         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8088         final &= ~DREF_SSC1_ENABLE;
8089
8090         if (has_panel) {
8091                 final |= DREF_SSC_SOURCE_ENABLE;
8092
8093                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8094                         final |= DREF_SSC1_ENABLE;
8095
8096                 if (has_cpu_edp) {
8097                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8098                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8099                         else
8100                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8101                 } else
8102                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8103         } else if (using_ssc_source) {
8104                 final |= DREF_SSC_SOURCE_ENABLE;
8105                 final |= DREF_SSC1_ENABLE;
8106         }
8107
8108         if (final == val)
8109                 return;
8110
8111         /* Always enable nonspread source */
8112         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8113
8114         if (has_ck505)
8115                 val |= DREF_NONSPREAD_CK505_ENABLE;
8116         else
8117                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8118
8119         if (has_panel) {
8120                 val &= ~DREF_SSC_SOURCE_MASK;
8121                 val |= DREF_SSC_SOURCE_ENABLE;
8122
8123                 /* SSC must be turned on before enabling the CPU output  */
8124                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8125                         DRM_DEBUG_KMS("Using SSC on panel\n");
8126                         val |= DREF_SSC1_ENABLE;
8127                 } else
8128                         val &= ~DREF_SSC1_ENABLE;
8129
8130                 /* Get SSC going before enabling the outputs */
8131                 I915_WRITE(PCH_DREF_CONTROL, val);
8132                 POSTING_READ(PCH_DREF_CONTROL);
8133                 udelay(200);
8134
8135                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8136
8137                 /* Enable CPU source on CPU attached eDP */
8138                 if (has_cpu_edp) {
8139                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8140                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8141                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8142                         } else
8143                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8144                 } else
8145                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8146
8147                 I915_WRITE(PCH_DREF_CONTROL, val);
8148                 POSTING_READ(PCH_DREF_CONTROL);
8149                 udelay(200);
8150         } else {
8151                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8152
8153                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8154
8155                 /* Turn off CPU output */
8156                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8157
8158                 I915_WRITE(PCH_DREF_CONTROL, val);
8159                 POSTING_READ(PCH_DREF_CONTROL);
8160                 udelay(200);
8161
8162                 if (!using_ssc_source) {
8163                         DRM_DEBUG_KMS("Disabling SSC source\n");
8164
8165                         /* Turn off the SSC source */
8166                         val &= ~DREF_SSC_SOURCE_MASK;
8167                         val |= DREF_SSC_SOURCE_DISABLE;
8168
8169                         /* Turn off SSC1 */
8170                         val &= ~DREF_SSC1_ENABLE;
8171
8172                         I915_WRITE(PCH_DREF_CONTROL, val);
8173                         POSTING_READ(PCH_DREF_CONTROL);
8174                         udelay(200);
8175                 }
8176         }
8177
8178         BUG_ON(val != final);
8179 }
8180
8181 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8182 {
8183         uint32_t tmp;
8184
8185         tmp = I915_READ(SOUTH_CHICKEN2);
8186         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8187         I915_WRITE(SOUTH_CHICKEN2, tmp);
8188
8189         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
8190                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8191                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8192
8193         tmp = I915_READ(SOUTH_CHICKEN2);
8194         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8195         I915_WRITE(SOUTH_CHICKEN2, tmp);
8196
8197         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
8198                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8199                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8200 }
8201
8202 /* WaMPhyProgramming:hsw */
8203 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8204 {
8205         uint32_t tmp;
8206
8207         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8208         tmp &= ~(0xFF << 24);
8209         tmp |= (0x12 << 24);
8210         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8211
8212         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8213         tmp |= (1 << 11);
8214         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8215
8216         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8217         tmp |= (1 << 11);
8218         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8219
8220         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8221         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8222         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8223
8224         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8225         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8226         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8227
8228         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8229         tmp &= ~(7 << 13);
8230         tmp |= (5 << 13);
8231         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8232
8233         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8234         tmp &= ~(7 << 13);
8235         tmp |= (5 << 13);
8236         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8237
8238         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8239         tmp &= ~0xFF;
8240         tmp |= 0x1C;
8241         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8242
8243         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8244         tmp &= ~0xFF;
8245         tmp |= 0x1C;
8246         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8247
8248         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8249         tmp &= ~(0xFF << 16);
8250         tmp |= (0x1C << 16);
8251         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8252
8253         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8254         tmp &= ~(0xFF << 16);
8255         tmp |= (0x1C << 16);
8256         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8257
8258         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8259         tmp |= (1 << 27);
8260         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8261
8262         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8263         tmp |= (1 << 27);
8264         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8265
8266         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8267         tmp &= ~(0xF << 28);
8268         tmp |= (4 << 28);
8269         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8270
8271         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8272         tmp &= ~(0xF << 28);
8273         tmp |= (4 << 28);
8274         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8275 }
8276
8277 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8278  * Programming" based on the parameters passed:
8279  * - Sequence to enable CLKOUT_DP
8280  * - Sequence to enable CLKOUT_DP without spread
8281  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8282  */
8283 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
8284                                  bool with_spread, bool with_fdi)
8285 {
8286         uint32_t reg, tmp;
8287
8288         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8289                 with_spread = true;
8290         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
8291             with_fdi, "LP PCH doesn't have FDI\n"))
8292                 with_fdi = false;
8293
8294         mutex_lock(&dev_priv->sb_lock);
8295
8296         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8297         tmp &= ~SBI_SSCCTL_DISABLE;
8298         tmp |= SBI_SSCCTL_PATHALT;
8299         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8300
8301         udelay(24);
8302
8303         if (with_spread) {
8304                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8305                 tmp &= ~SBI_SSCCTL_PATHALT;
8306                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8307
8308                 if (with_fdi) {
8309                         lpt_reset_fdi_mphy(dev_priv);
8310                         lpt_program_fdi_mphy(dev_priv);
8311                 }
8312         }
8313
8314         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8315         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8316         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8317         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8318
8319         mutex_unlock(&dev_priv->sb_lock);
8320 }
8321
8322 /* Sequence to disable CLKOUT_DP */
8323 static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
8324 {
8325         uint32_t reg, tmp;
8326
8327         mutex_lock(&dev_priv->sb_lock);
8328
8329         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8330         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8331         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8332         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8333
8334         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8335         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8336                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8337                         tmp |= SBI_SSCCTL_PATHALT;
8338                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8339                         udelay(32);
8340                 }
8341                 tmp |= SBI_SSCCTL_DISABLE;
8342                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8343         }
8344
8345         mutex_unlock(&dev_priv->sb_lock);
8346 }
8347
8348 #define BEND_IDX(steps) ((50 + (steps)) / 5)
8349
8350 static const uint16_t sscdivintphase[] = {
8351         [BEND_IDX( 50)] = 0x3B23,
8352         [BEND_IDX( 45)] = 0x3B23,
8353         [BEND_IDX( 40)] = 0x3C23,
8354         [BEND_IDX( 35)] = 0x3C23,
8355         [BEND_IDX( 30)] = 0x3D23,
8356         [BEND_IDX( 25)] = 0x3D23,
8357         [BEND_IDX( 20)] = 0x3E23,
8358         [BEND_IDX( 15)] = 0x3E23,
8359         [BEND_IDX( 10)] = 0x3F23,
8360         [BEND_IDX(  5)] = 0x3F23,
8361         [BEND_IDX(  0)] = 0x0025,
8362         [BEND_IDX( -5)] = 0x0025,
8363         [BEND_IDX(-10)] = 0x0125,
8364         [BEND_IDX(-15)] = 0x0125,
8365         [BEND_IDX(-20)] = 0x0225,
8366         [BEND_IDX(-25)] = 0x0225,
8367         [BEND_IDX(-30)] = 0x0325,
8368         [BEND_IDX(-35)] = 0x0325,
8369         [BEND_IDX(-40)] = 0x0425,
8370         [BEND_IDX(-45)] = 0x0425,
8371         [BEND_IDX(-50)] = 0x0525,
8372 };
8373
8374 /*
8375  * Bend CLKOUT_DP
8376  * steps -50 to 50 inclusive, in steps of 5
8377  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
8378  * change in clock period = -(steps / 10) * 5.787 ps
8379  */
8380 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
8381 {
8382         uint32_t tmp;
8383         int idx = BEND_IDX(steps);
8384
8385         if (WARN_ON(steps % 5 != 0))
8386                 return;
8387
8388         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
8389                 return;
8390
8391         mutex_lock(&dev_priv->sb_lock);
8392
8393         if (steps % 10 != 0)
8394                 tmp = 0xAAAAAAAB;
8395         else
8396                 tmp = 0x00000000;
8397         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
8398
8399         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
8400         tmp &= 0xffff0000;
8401         tmp |= sscdivintphase[idx];
8402         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
8403
8404         mutex_unlock(&dev_priv->sb_lock);
8405 }
8406
8407 #undef BEND_IDX
8408
8409 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
8410 {
8411         struct intel_encoder *encoder;
8412         bool has_vga = false;
8413
8414         for_each_intel_encoder(&dev_priv->drm, encoder) {
8415                 switch (encoder->type) {
8416                 case INTEL_OUTPUT_ANALOG:
8417                         has_vga = true;
8418                         break;
8419                 default:
8420                         break;
8421                 }
8422         }
8423
8424         if (has_vga) {
8425                 lpt_bend_clkout_dp(dev_priv, 0);
8426                 lpt_enable_clkout_dp(dev_priv, true, true);
8427         } else {
8428                 lpt_disable_clkout_dp(dev_priv);
8429         }
8430 }
8431
8432 /*
8433  * Initialize reference clocks when the driver loads
8434  */
8435 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
8436 {
8437         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
8438                 ironlake_init_pch_refclk(dev_priv);
8439         else if (HAS_PCH_LPT(dev_priv))
8440                 lpt_init_pch_refclk(dev_priv);
8441 }
8442
8443 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
8444 {
8445         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8446         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8447         enum pipe pipe = crtc->pipe;
8448         uint32_t val;
8449
8450         val = 0;
8451
8452         switch (crtc_state->pipe_bpp) {
8453         case 18:
8454                 val |= PIPECONF_6BPC;
8455                 break;
8456         case 24:
8457                 val |= PIPECONF_8BPC;
8458                 break;
8459         case 30:
8460                 val |= PIPECONF_10BPC;
8461                 break;
8462         case 36:
8463                 val |= PIPECONF_12BPC;
8464                 break;
8465         default:
8466                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8467                 BUG();
8468         }
8469
8470         if (crtc_state->dither)
8471                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8472
8473         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8474                 val |= PIPECONF_INTERLACED_ILK;
8475         else
8476                 val |= PIPECONF_PROGRESSIVE;
8477
8478         if (crtc_state->limited_color_range)
8479                 val |= PIPECONF_COLOR_RANGE_SELECT;
8480
8481         I915_WRITE(PIPECONF(pipe), val);
8482         POSTING_READ(PIPECONF(pipe));
8483 }
8484
8485 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
8486 {
8487         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8488         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8489         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8490         u32 val = 0;
8491
8492         if (IS_HASWELL(dev_priv) && crtc_state->dither)
8493                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8494
8495         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8496                 val |= PIPECONF_INTERLACED_ILK;
8497         else
8498                 val |= PIPECONF_PROGRESSIVE;
8499
8500         I915_WRITE(PIPECONF(cpu_transcoder), val);
8501         POSTING_READ(PIPECONF(cpu_transcoder));
8502 }
8503
8504 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state)
8505 {
8506         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
8507         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
8508
8509         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8510                 u32 val = 0;
8511
8512                 switch (crtc_state->pipe_bpp) {
8513                 case 18:
8514                         val |= PIPEMISC_DITHER_6_BPC;
8515                         break;
8516                 case 24:
8517                         val |= PIPEMISC_DITHER_8_BPC;
8518                         break;
8519                 case 30:
8520                         val |= PIPEMISC_DITHER_10_BPC;
8521                         break;
8522                 case 36:
8523                         val |= PIPEMISC_DITHER_12_BPC;
8524                         break;
8525                 default:
8526                         /* Case prevented by pipe_config_set_bpp. */
8527                         BUG();
8528                 }
8529
8530                 if (crtc_state->dither)
8531                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8532
8533                 if (crtc_state->ycbcr420) {
8534                         val |= PIPEMISC_OUTPUT_COLORSPACE_YUV |
8535                                 PIPEMISC_YUV420_ENABLE |
8536                                 PIPEMISC_YUV420_MODE_FULL_BLEND;
8537                 }
8538
8539                 I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
8540         }
8541 }
8542
8543 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8544 {
8545         /*
8546          * Account for spread spectrum to avoid
8547          * oversubscribing the link. Max center spread
8548          * is 2.5%; use 5% for safety's sake.
8549          */
8550         u32 bps = target_clock * bpp * 21 / 20;
8551         return DIV_ROUND_UP(bps, link_bw * 8);
8552 }
8553
8554 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8555 {
8556         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8557 }
8558
8559 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8560                                   struct intel_crtc_state *crtc_state,
8561                                   struct dpll *reduced_clock)
8562 {
8563         struct drm_crtc *crtc = &intel_crtc->base;
8564         struct drm_device *dev = crtc->dev;
8565         struct drm_i915_private *dev_priv = to_i915(dev);
8566         u32 dpll, fp, fp2;
8567         int factor;
8568
8569         /* Enable autotuning of the PLL clock (if permissible) */
8570         factor = 21;
8571         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8572                 if ((intel_panel_use_ssc(dev_priv) &&
8573                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8574                     (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
8575                         factor = 25;
8576         } else if (crtc_state->sdvo_tv_clock)
8577                 factor = 20;
8578
8579         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8580
8581         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8582                 fp |= FP_CB_TUNE;
8583
8584         if (reduced_clock) {
8585                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
8586
8587                 if (reduced_clock->m < factor * reduced_clock->n)
8588                         fp2 |= FP_CB_TUNE;
8589         } else {
8590                 fp2 = fp;
8591         }
8592
8593         dpll = 0;
8594
8595         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8596                 dpll |= DPLLB_MODE_LVDS;
8597         else
8598                 dpll |= DPLLB_MODE_DAC_SERIAL;
8599
8600         dpll |= (crtc_state->pixel_multiplier - 1)
8601                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8602
8603         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8604             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8605                 dpll |= DPLL_SDVO_HIGH_SPEED;
8606
8607         if (intel_crtc_has_dp_encoder(crtc_state))
8608                 dpll |= DPLL_SDVO_HIGH_SPEED;
8609
8610         /*
8611          * The high speed IO clock is only really required for
8612          * SDVO/HDMI/DP, but we also enable it for CRT to make it
8613          * possible to share the DPLL between CRT and HDMI. Enabling
8614          * the clock needlessly does no real harm, except use up a
8615          * bit of power potentially.
8616          *
8617          * We'll limit this to IVB with 3 pipes, since it has only two
8618          * DPLLs and so DPLL sharing is the only way to get three pipes
8619          * driving PCH ports at the same time. On SNB we could do this,
8620          * and potentially avoid enabling the second DPLL, but it's not
8621          * clear if it''s a win or loss power wise. No point in doing
8622          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
8623          */
8624         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
8625             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
8626                 dpll |= DPLL_SDVO_HIGH_SPEED;
8627
8628         /* compute bitmask from p1 value */
8629         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8630         /* also FPA1 */
8631         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8632
8633         switch (crtc_state->dpll.p2) {
8634         case 5:
8635                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8636                 break;
8637         case 7:
8638                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8639                 break;
8640         case 10:
8641                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8642                 break;
8643         case 14:
8644                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8645                 break;
8646         }
8647
8648         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8649             intel_panel_use_ssc(dev_priv))
8650                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8651         else
8652                 dpll |= PLL_REF_INPUT_DREFCLK;
8653
8654         dpll |= DPLL_VCO_ENABLE;
8655
8656         crtc_state->dpll_hw_state.dpll = dpll;
8657         crtc_state->dpll_hw_state.fp0 = fp;
8658         crtc_state->dpll_hw_state.fp1 = fp2;
8659 }
8660
8661 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8662                                        struct intel_crtc_state *crtc_state)
8663 {
8664         struct drm_device *dev = crtc->base.dev;
8665         struct drm_i915_private *dev_priv = to_i915(dev);
8666         const struct intel_limit *limit;
8667         int refclk = 120000;
8668
8669         memset(&crtc_state->dpll_hw_state, 0,
8670                sizeof(crtc_state->dpll_hw_state));
8671
8672         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8673         if (!crtc_state->has_pch_encoder)
8674                 return 0;
8675
8676         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8677                 if (intel_panel_use_ssc(dev_priv)) {
8678                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8679                                       dev_priv->vbt.lvds_ssc_freq);
8680                         refclk = dev_priv->vbt.lvds_ssc_freq;
8681                 }
8682
8683                 if (intel_is_dual_link_lvds(dev)) {
8684                         if (refclk == 100000)
8685                                 limit = &intel_limits_ironlake_dual_lvds_100m;
8686                         else
8687                                 limit = &intel_limits_ironlake_dual_lvds;
8688                 } else {
8689                         if (refclk == 100000)
8690                                 limit = &intel_limits_ironlake_single_lvds_100m;
8691                         else
8692                                 limit = &intel_limits_ironlake_single_lvds;
8693                 }
8694         } else {
8695                 limit = &intel_limits_ironlake_dac;
8696         }
8697
8698         if (!crtc_state->clock_set &&
8699             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8700                                 refclk, NULL, &crtc_state->dpll)) {
8701                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8702                 return -EINVAL;
8703         }
8704
8705         ironlake_compute_dpll(crtc, crtc_state, NULL);
8706
8707         if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
8708                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
8709                               pipe_name(crtc->pipe));
8710                 return -EINVAL;
8711         }
8712
8713         return 0;
8714 }
8715
8716 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8717                                          struct intel_link_m_n *m_n)
8718 {
8719         struct drm_device *dev = crtc->base.dev;
8720         struct drm_i915_private *dev_priv = to_i915(dev);
8721         enum pipe pipe = crtc->pipe;
8722
8723         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8724         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8725         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8726                 & ~TU_SIZE_MASK;
8727         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8728         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8729                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8730 }
8731
8732 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8733                                          enum transcoder transcoder,
8734                                          struct intel_link_m_n *m_n,
8735                                          struct intel_link_m_n *m2_n2)
8736 {
8737         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8738         enum pipe pipe = crtc->pipe;
8739
8740         if (INTEL_GEN(dev_priv) >= 5) {
8741                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
8742                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
8743                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
8744                         & ~TU_SIZE_MASK;
8745                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
8746                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
8747                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8748                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
8749                  * gen < 8) and if DRRS is supported (to make sure the
8750                  * registers are not unnecessarily read).
8751                  */
8752                 if (m2_n2 && INTEL_GEN(dev_priv) < 8 &&
8753                         crtc->config->has_drrs) {
8754                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
8755                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
8756                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
8757                                         & ~TU_SIZE_MASK;
8758                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
8759                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
8760                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8761                 }
8762         } else {
8763                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
8764                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
8765                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
8766                         & ~TU_SIZE_MASK;
8767                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
8768                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
8769                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8770         }
8771 }
8772
8773 void intel_dp_get_m_n(struct intel_crtc *crtc,
8774                       struct intel_crtc_state *pipe_config)
8775 {
8776         if (pipe_config->has_pch_encoder)
8777                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8778         else
8779                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8780                                              &pipe_config->dp_m_n,
8781                                              &pipe_config->dp_m2_n2);
8782 }
8783
8784 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
8785                                         struct intel_crtc_state *pipe_config)
8786 {
8787         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8788                                      &pipe_config->fdi_m_n, NULL);
8789 }
8790
8791 static void skylake_get_pfit_config(struct intel_crtc *crtc,
8792                                     struct intel_crtc_state *pipe_config)
8793 {
8794         struct drm_device *dev = crtc->base.dev;
8795         struct drm_i915_private *dev_priv = to_i915(dev);
8796         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
8797         uint32_t ps_ctrl = 0;
8798         int id = -1;
8799         int i;
8800
8801         /* find scaler attached to this pipe */
8802         for (i = 0; i < crtc->num_scalers; i++) {
8803                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
8804                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
8805                         id = i;
8806                         pipe_config->pch_pfit.enabled = true;
8807                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
8808                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
8809                         break;
8810                 }
8811         }
8812
8813         scaler_state->scaler_id = id;
8814         if (id >= 0) {
8815                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8816         } else {
8817                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8818         }
8819 }
8820
8821 static void
8822 skylake_get_initial_plane_config(struct intel_crtc *crtc,
8823                                  struct intel_initial_plane_config *plane_config)
8824 {
8825         struct drm_device *dev = crtc->base.dev;
8826         struct drm_i915_private *dev_priv = to_i915(dev);
8827         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8828         enum plane_id plane_id = plane->id;
8829         enum pipe pipe;
8830         u32 val, base, offset, stride_mult, tiling, alpha;
8831         int fourcc, pixel_format;
8832         unsigned int aligned_height;
8833         struct drm_framebuffer *fb;
8834         struct intel_framebuffer *intel_fb;
8835
8836         if (!plane->get_hw_state(plane, &pipe))
8837                 return;
8838
8839         WARN_ON(pipe != crtc->pipe);
8840
8841         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8842         if (!intel_fb) {
8843                 DRM_DEBUG_KMS("failed to alloc fb\n");
8844                 return;
8845         }
8846
8847         fb = &intel_fb->base;
8848
8849         fb->dev = dev;
8850
8851         val = I915_READ(PLANE_CTL(pipe, plane_id));
8852
8853         if (INTEL_GEN(dev_priv) >= 11)
8854                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
8855         else
8856                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
8857
8858         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
8859                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
8860                 alpha &= PLANE_COLOR_ALPHA_MASK;
8861         } else {
8862                 alpha = val & PLANE_CTL_ALPHA_MASK;
8863         }
8864
8865         fourcc = skl_format_to_fourcc(pixel_format,
8866                                       val & PLANE_CTL_ORDER_RGBX, alpha);
8867         fb->format = drm_format_info(fourcc);
8868
8869         tiling = val & PLANE_CTL_TILED_MASK;
8870         switch (tiling) {
8871         case PLANE_CTL_TILED_LINEAR:
8872                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
8873                 break;
8874         case PLANE_CTL_TILED_X:
8875                 plane_config->tiling = I915_TILING_X;
8876                 fb->modifier = I915_FORMAT_MOD_X_TILED;
8877                 break;
8878         case PLANE_CTL_TILED_Y:
8879                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8880                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
8881                 else
8882                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
8883                 break;
8884         case PLANE_CTL_TILED_YF:
8885                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8886                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
8887                 else
8888                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
8889                 break;
8890         default:
8891                 MISSING_CASE(tiling);
8892                 goto error;
8893         }
8894
8895         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
8896         plane_config->base = base;
8897
8898         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
8899
8900         val = I915_READ(PLANE_SIZE(pipe, plane_id));
8901         fb->height = ((val >> 16) & 0xfff) + 1;
8902         fb->width = ((val >> 0) & 0x1fff) + 1;
8903
8904         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
8905         stride_mult = intel_fb_stride_alignment(fb, 0);
8906         fb->pitches[0] = (val & 0x3ff) * stride_mult;
8907
8908         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8909
8910         plane_config->size = fb->pitches[0] * aligned_height;
8911
8912         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8913                       crtc->base.name, plane->base.name, fb->width, fb->height,
8914                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8915                       plane_config->size);
8916
8917         plane_config->fb = intel_fb;
8918         return;
8919
8920 error:
8921         kfree(intel_fb);
8922 }
8923
8924 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
8925                                      struct intel_crtc_state *pipe_config)
8926 {
8927         struct drm_device *dev = crtc->base.dev;
8928         struct drm_i915_private *dev_priv = to_i915(dev);
8929         uint32_t tmp;
8930
8931         tmp = I915_READ(PF_CTL(crtc->pipe));
8932
8933         if (tmp & PF_ENABLE) {
8934                 pipe_config->pch_pfit.enabled = true;
8935                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
8936                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
8937
8938                 /* We currently do not free assignements of panel fitters on
8939                  * ivb/hsw (since we don't use the higher upscaling modes which
8940                  * differentiates them) so just WARN about this case for now. */
8941                 if (IS_GEN7(dev_priv)) {
8942                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
8943                                 PF_PIPE_SEL_IVB(crtc->pipe));
8944                 }
8945         }
8946 }
8947
8948 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
8949                                      struct intel_crtc_state *pipe_config)
8950 {
8951         struct drm_device *dev = crtc->base.dev;
8952         struct drm_i915_private *dev_priv = to_i915(dev);
8953         enum intel_display_power_domain power_domain;
8954         uint32_t tmp;
8955         bool ret;
8956
8957         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8958         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
8959                 return false;
8960
8961         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8962         pipe_config->shared_dpll = NULL;
8963
8964         ret = false;
8965         tmp = I915_READ(PIPECONF(crtc->pipe));
8966         if (!(tmp & PIPECONF_ENABLE))
8967                 goto out;
8968
8969         switch (tmp & PIPECONF_BPC_MASK) {
8970         case PIPECONF_6BPC:
8971                 pipe_config->pipe_bpp = 18;
8972                 break;
8973         case PIPECONF_8BPC:
8974                 pipe_config->pipe_bpp = 24;
8975                 break;
8976         case PIPECONF_10BPC:
8977                 pipe_config->pipe_bpp = 30;
8978                 break;
8979         case PIPECONF_12BPC:
8980                 pipe_config->pipe_bpp = 36;
8981                 break;
8982         default:
8983                 break;
8984         }
8985
8986         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
8987                 pipe_config->limited_color_range = true;
8988
8989         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
8990                 struct intel_shared_dpll *pll;
8991                 enum intel_dpll_id pll_id;
8992
8993                 pipe_config->has_pch_encoder = true;
8994
8995                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
8996                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
8997                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
8998
8999                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9000
9001                 if (HAS_PCH_IBX(dev_priv)) {
9002                         /*
9003                          * The pipe->pch transcoder and pch transcoder->pll
9004                          * mapping is fixed.
9005                          */
9006                         pll_id = (enum intel_dpll_id) crtc->pipe;
9007                 } else {
9008                         tmp = I915_READ(PCH_DPLL_SEL);
9009                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9010                                 pll_id = DPLL_ID_PCH_PLL_B;
9011                         else
9012                                 pll_id= DPLL_ID_PCH_PLL_A;
9013                 }
9014
9015                 pipe_config->shared_dpll =
9016                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
9017                 pll = pipe_config->shared_dpll;
9018
9019                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9020                                                 &pipe_config->dpll_hw_state));
9021
9022                 tmp = pipe_config->dpll_hw_state.dpll;
9023                 pipe_config->pixel_multiplier =
9024                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9025                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9026
9027                 ironlake_pch_clock_get(crtc, pipe_config);
9028         } else {
9029                 pipe_config->pixel_multiplier = 1;
9030         }
9031
9032         intel_get_pipe_timings(crtc, pipe_config);
9033         intel_get_pipe_src_size(crtc, pipe_config);
9034
9035         ironlake_get_pfit_config(crtc, pipe_config);
9036
9037         ret = true;
9038
9039 out:
9040         intel_display_power_put(dev_priv, power_domain);
9041
9042         return ret;
9043 }
9044
9045 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9046 {
9047         struct drm_device *dev = &dev_priv->drm;
9048         struct intel_crtc *crtc;
9049
9050         for_each_intel_crtc(dev, crtc)
9051                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9052                      pipe_name(crtc->pipe));
9053
9054         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL2),
9055                         "Display power well on\n");
9056         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9057         I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9058         I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9059         I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
9060         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9061              "CPU PWM1 enabled\n");
9062         if (IS_HASWELL(dev_priv))
9063                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9064                      "CPU PWM2 enabled\n");
9065         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9066              "PCH PWM1 enabled\n");
9067         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9068              "Utility pin enabled\n");
9069         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9070
9071         /*
9072          * In theory we can still leave IRQs enabled, as long as only the HPD
9073          * interrupts remain enabled. We used to check for that, but since it's
9074          * gen-specific and since we only disable LCPLL after we fully disable
9075          * the interrupts, the check below should be enough.
9076          */
9077         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9078 }
9079
9080 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9081 {
9082         if (IS_HASWELL(dev_priv))
9083                 return I915_READ(D_COMP_HSW);
9084         else
9085                 return I915_READ(D_COMP_BDW);
9086 }
9087
9088 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9089 {
9090         if (IS_HASWELL(dev_priv)) {
9091                 mutex_lock(&dev_priv->pcu_lock);
9092                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9093                                             val))
9094                         DRM_DEBUG_KMS("Failed to write to D_COMP\n");
9095                 mutex_unlock(&dev_priv->pcu_lock);
9096         } else {
9097                 I915_WRITE(D_COMP_BDW, val);
9098                 POSTING_READ(D_COMP_BDW);
9099         }
9100 }
9101
9102 /*
9103  * This function implements pieces of two sequences from BSpec:
9104  * - Sequence for display software to disable LCPLL
9105  * - Sequence for display software to allow package C8+
9106  * The steps implemented here are just the steps that actually touch the LCPLL
9107  * register. Callers should take care of disabling all the display engine
9108  * functions, doing the mode unset, fixing interrupts, etc.
9109  */
9110 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9111                               bool switch_to_fclk, bool allow_power_down)
9112 {
9113         uint32_t val;
9114
9115         assert_can_disable_lcpll(dev_priv);
9116
9117         val = I915_READ(LCPLL_CTL);
9118
9119         if (switch_to_fclk) {
9120                 val |= LCPLL_CD_SOURCE_FCLK;
9121                 I915_WRITE(LCPLL_CTL, val);
9122
9123                 if (wait_for_us(I915_READ(LCPLL_CTL) &
9124                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
9125                         DRM_ERROR("Switching to FCLK failed\n");
9126
9127                 val = I915_READ(LCPLL_CTL);
9128         }
9129
9130         val |= LCPLL_PLL_DISABLE;
9131         I915_WRITE(LCPLL_CTL, val);
9132         POSTING_READ(LCPLL_CTL);
9133
9134         if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
9135                 DRM_ERROR("LCPLL still locked\n");
9136
9137         val = hsw_read_dcomp(dev_priv);
9138         val |= D_COMP_COMP_DISABLE;
9139         hsw_write_dcomp(dev_priv, val);
9140         ndelay(100);
9141
9142         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9143                      1))
9144                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9145
9146         if (allow_power_down) {
9147                 val = I915_READ(LCPLL_CTL);
9148                 val |= LCPLL_POWER_DOWN_ALLOW;
9149                 I915_WRITE(LCPLL_CTL, val);
9150                 POSTING_READ(LCPLL_CTL);
9151         }
9152 }
9153
9154 /*
9155  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9156  * source.
9157  */
9158 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9159 {
9160         uint32_t val;
9161
9162         val = I915_READ(LCPLL_CTL);
9163
9164         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9165                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9166                 return;
9167
9168         /*
9169          * Make sure we're not on PC8 state before disabling PC8, otherwise
9170          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9171          */
9172         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9173
9174         if (val & LCPLL_POWER_DOWN_ALLOW) {
9175                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9176                 I915_WRITE(LCPLL_CTL, val);
9177                 POSTING_READ(LCPLL_CTL);
9178         }
9179
9180         val = hsw_read_dcomp(dev_priv);
9181         val |= D_COMP_COMP_FORCE;
9182         val &= ~D_COMP_COMP_DISABLE;
9183         hsw_write_dcomp(dev_priv, val);
9184
9185         val = I915_READ(LCPLL_CTL);
9186         val &= ~LCPLL_PLL_DISABLE;
9187         I915_WRITE(LCPLL_CTL, val);
9188
9189         if (intel_wait_for_register(dev_priv,
9190                                     LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
9191                                     5))
9192                 DRM_ERROR("LCPLL not locked yet\n");
9193
9194         if (val & LCPLL_CD_SOURCE_FCLK) {
9195                 val = I915_READ(LCPLL_CTL);
9196                 val &= ~LCPLL_CD_SOURCE_FCLK;
9197                 I915_WRITE(LCPLL_CTL, val);
9198
9199                 if (wait_for_us((I915_READ(LCPLL_CTL) &
9200                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9201                         DRM_ERROR("Switching back to LCPLL failed\n");
9202         }
9203
9204         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9205
9206         intel_update_cdclk(dev_priv);
9207         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
9208 }
9209
9210 /*
9211  * Package states C8 and deeper are really deep PC states that can only be
9212  * reached when all the devices on the system allow it, so even if the graphics
9213  * device allows PC8+, it doesn't mean the system will actually get to these
9214  * states. Our driver only allows PC8+ when going into runtime PM.
9215  *
9216  * The requirements for PC8+ are that all the outputs are disabled, the power
9217  * well is disabled and most interrupts are disabled, and these are also
9218  * requirements for runtime PM. When these conditions are met, we manually do
9219  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9220  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9221  * hang the machine.
9222  *
9223  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9224  * the state of some registers, so when we come back from PC8+ we need to
9225  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9226  * need to take care of the registers kept by RC6. Notice that this happens even
9227  * if we don't put the device in PCI D3 state (which is what currently happens
9228  * because of the runtime PM support).
9229  *
9230  * For more, read "Display Sequences for Package C8" on the hardware
9231  * documentation.
9232  */
9233 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9234 {
9235         uint32_t val;
9236
9237         DRM_DEBUG_KMS("Enabling package C8+\n");
9238
9239         if (HAS_PCH_LPT_LP(dev_priv)) {
9240                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9241                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9242                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9243         }
9244
9245         lpt_disable_clkout_dp(dev_priv);
9246         hsw_disable_lcpll(dev_priv, true, true);
9247 }
9248
9249 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9250 {
9251         uint32_t val;
9252
9253         DRM_DEBUG_KMS("Disabling package C8+\n");
9254
9255         hsw_restore_lcpll(dev_priv);
9256         lpt_init_pch_refclk(dev_priv);
9257
9258         if (HAS_PCH_LPT_LP(dev_priv)) {
9259                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9260                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9261                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9262         }
9263 }
9264
9265 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9266                                       struct intel_crtc_state *crtc_state)
9267 {
9268         struct intel_atomic_state *state =
9269                 to_intel_atomic_state(crtc_state->base.state);
9270
9271         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) {
9272                 struct intel_encoder *encoder =
9273                         intel_get_crtc_new_encoder(state, crtc_state);
9274
9275                 if (!intel_get_shared_dpll(crtc, crtc_state, encoder)) {
9276                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9277                                       pipe_name(crtc->pipe));
9278                         return -EINVAL;
9279                 }
9280         }
9281
9282         return 0;
9283 }
9284
9285 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
9286                                    enum port port,
9287                                    struct intel_crtc_state *pipe_config)
9288 {
9289         enum intel_dpll_id id;
9290         u32 temp;
9291
9292         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9293         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9294
9295         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
9296                 return;
9297
9298         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9299 }
9300
9301 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
9302                                 enum port port,
9303                                 struct intel_crtc_state *pipe_config)
9304 {
9305         enum intel_dpll_id id;
9306         u32 temp;
9307
9308         /* TODO: TBT pll not implemented. */
9309         switch (port) {
9310         case PORT_A:
9311         case PORT_B:
9312                 temp = I915_READ(DPCLKA_CFGCR0_ICL) &
9313                        DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9314                 id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9315
9316                 if (WARN_ON(id != DPLL_ID_ICL_DPLL0 && id != DPLL_ID_ICL_DPLL1))
9317                         return;
9318                 break;
9319         case PORT_C:
9320                 id = DPLL_ID_ICL_MGPLL1;
9321                 break;
9322         case PORT_D:
9323                 id = DPLL_ID_ICL_MGPLL2;
9324                 break;
9325         case PORT_E:
9326                 id = DPLL_ID_ICL_MGPLL3;
9327                 break;
9328         case PORT_F:
9329                 id = DPLL_ID_ICL_MGPLL4;
9330                 break;
9331         default:
9332                 MISSING_CASE(port);
9333                 return;
9334         }
9335
9336         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9337 }
9338
9339 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9340                                 enum port port,
9341                                 struct intel_crtc_state *pipe_config)
9342 {
9343         enum intel_dpll_id id;
9344
9345         switch (port) {
9346         case PORT_A:
9347                 id = DPLL_ID_SKL_DPLL0;
9348                 break;
9349         case PORT_B:
9350                 id = DPLL_ID_SKL_DPLL1;
9351                 break;
9352         case PORT_C:
9353                 id = DPLL_ID_SKL_DPLL2;
9354                 break;
9355         default:
9356                 DRM_ERROR("Incorrect port type\n");
9357                 return;
9358         }
9359
9360         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9361 }
9362
9363 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9364                                 enum port port,
9365                                 struct intel_crtc_state *pipe_config)
9366 {
9367         enum intel_dpll_id id;
9368         u32 temp;
9369
9370         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9371         id = temp >> (port * 3 + 1);
9372
9373         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
9374                 return;
9375
9376         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9377 }
9378
9379 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9380                                 enum port port,
9381                                 struct intel_crtc_state *pipe_config)
9382 {
9383         enum intel_dpll_id id;
9384         uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9385
9386         switch (ddi_pll_sel) {
9387         case PORT_CLK_SEL_WRPLL1:
9388                 id = DPLL_ID_WRPLL1;
9389                 break;
9390         case PORT_CLK_SEL_WRPLL2:
9391                 id = DPLL_ID_WRPLL2;
9392                 break;
9393         case PORT_CLK_SEL_SPLL:
9394                 id = DPLL_ID_SPLL;
9395                 break;
9396         case PORT_CLK_SEL_LCPLL_810:
9397                 id = DPLL_ID_LCPLL_810;
9398                 break;
9399         case PORT_CLK_SEL_LCPLL_1350:
9400                 id = DPLL_ID_LCPLL_1350;
9401                 break;
9402         case PORT_CLK_SEL_LCPLL_2700:
9403                 id = DPLL_ID_LCPLL_2700;
9404                 break;
9405         default:
9406                 MISSING_CASE(ddi_pll_sel);
9407                 /* fall through */
9408         case PORT_CLK_SEL_NONE:
9409                 return;
9410         }
9411
9412         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9413 }
9414
9415 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
9416                                      struct intel_crtc_state *pipe_config,
9417                                      u64 *power_domain_mask)
9418 {
9419         struct drm_device *dev = crtc->base.dev;
9420         struct drm_i915_private *dev_priv = to_i915(dev);
9421         enum intel_display_power_domain power_domain;
9422         u32 tmp;
9423
9424         /*
9425          * The pipe->transcoder mapping is fixed with the exception of the eDP
9426          * transcoder handled below.
9427          */
9428         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9429
9430         /*
9431          * XXX: Do intel_display_power_get_if_enabled before reading this (for
9432          * consistency and less surprising code; it's in always on power).
9433          */
9434         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
9435         if (tmp & TRANS_DDI_FUNC_ENABLE) {
9436                 enum pipe trans_edp_pipe;
9437                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9438                 default:
9439                         WARN(1, "unknown pipe linked to edp transcoder\n");
9440                         /* fall through */
9441                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9442                 case TRANS_DDI_EDP_INPUT_A_ON:
9443                         trans_edp_pipe = PIPE_A;
9444                         break;
9445                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9446                         trans_edp_pipe = PIPE_B;
9447                         break;
9448                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9449                         trans_edp_pipe = PIPE_C;
9450                         break;
9451                 }
9452
9453                 if (trans_edp_pipe == crtc->pipe)
9454                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
9455         }
9456
9457         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
9458         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9459                 return false;
9460         *power_domain_mask |= BIT_ULL(power_domain);
9461
9462         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9463
9464         return tmp & PIPECONF_ENABLE;
9465 }
9466
9467 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
9468                                          struct intel_crtc_state *pipe_config,
9469                                          u64 *power_domain_mask)
9470 {
9471         struct drm_device *dev = crtc->base.dev;
9472         struct drm_i915_private *dev_priv = to_i915(dev);
9473         enum intel_display_power_domain power_domain;
9474         enum port port;
9475         enum transcoder cpu_transcoder;
9476         u32 tmp;
9477
9478         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
9479                 if (port == PORT_A)
9480                         cpu_transcoder = TRANSCODER_DSI_A;
9481                 else
9482                         cpu_transcoder = TRANSCODER_DSI_C;
9483
9484                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
9485                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9486                         continue;
9487                 *power_domain_mask |= BIT_ULL(power_domain);
9488
9489                 /*
9490                  * The PLL needs to be enabled with a valid divider
9491                  * configuration, otherwise accessing DSI registers will hang
9492                  * the machine. See BSpec North Display Engine
9493                  * registers/MIPI[BXT]. We can break out here early, since we
9494                  * need the same DSI PLL to be enabled for both DSI ports.
9495                  */
9496                 if (!bxt_dsi_pll_is_enabled(dev_priv))
9497                         break;
9498
9499                 /* XXX: this works for video mode only */
9500                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
9501                 if (!(tmp & DPI_ENABLE))
9502                         continue;
9503
9504                 tmp = I915_READ(MIPI_CTRL(port));
9505                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
9506                         continue;
9507
9508                 pipe_config->cpu_transcoder = cpu_transcoder;
9509                 break;
9510         }
9511
9512         return transcoder_is_dsi(pipe_config->cpu_transcoder);
9513 }
9514
9515 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9516                                        struct intel_crtc_state *pipe_config)
9517 {
9518         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9519         struct intel_shared_dpll *pll;
9520         enum port port;
9521         uint32_t tmp;
9522
9523         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9524
9525         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9526
9527         if (IS_ICELAKE(dev_priv))
9528                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
9529         else if (IS_CANNONLAKE(dev_priv))
9530                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
9531         else if (IS_GEN9_BC(dev_priv))
9532                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9533         else if (IS_GEN9_LP(dev_priv))
9534                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9535         else
9536                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9537
9538         pll = pipe_config->shared_dpll;
9539         if (pll) {
9540                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9541                                                 &pipe_config->dpll_hw_state));
9542         }
9543
9544         /*
9545          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9546          * DDI E. So just check whether this pipe is wired to DDI E and whether
9547          * the PCH transcoder is on.
9548          */
9549         if (INTEL_GEN(dev_priv) < 9 &&
9550             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9551                 pipe_config->has_pch_encoder = true;
9552
9553                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9554                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9555                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9556
9557                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9558         }
9559 }
9560
9561 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9562                                     struct intel_crtc_state *pipe_config)
9563 {
9564         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9565         enum intel_display_power_domain power_domain;
9566         u64 power_domain_mask;
9567         bool active;
9568
9569         intel_crtc_init_scalers(crtc, pipe_config);
9570
9571         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9572         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9573                 return false;
9574         power_domain_mask = BIT_ULL(power_domain);
9575
9576         pipe_config->shared_dpll = NULL;
9577
9578         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
9579
9580         if (IS_GEN9_LP(dev_priv) &&
9581             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
9582                 WARN_ON(active);
9583                 active = true;
9584         }
9585
9586         if (!active)
9587                 goto out;
9588
9589         if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9590                 haswell_get_ddi_port_state(crtc, pipe_config);
9591                 intel_get_pipe_timings(crtc, pipe_config);
9592         }
9593
9594         intel_get_pipe_src_size(crtc, pipe_config);
9595
9596         pipe_config->gamma_mode =
9597                 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
9598
9599         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
9600                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
9601                 bool clrspace_yuv = tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV;
9602
9603                 if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
9604                         bool blend_mode_420 = tmp &
9605                                               PIPEMISC_YUV420_MODE_FULL_BLEND;
9606
9607                         pipe_config->ycbcr420 = tmp & PIPEMISC_YUV420_ENABLE;
9608                         if (pipe_config->ycbcr420 != clrspace_yuv ||
9609                             pipe_config->ycbcr420 != blend_mode_420)
9610                                 DRM_DEBUG_KMS("Bad 4:2:0 mode (%08x)\n", tmp);
9611                 } else if (clrspace_yuv) {
9612                         DRM_DEBUG_KMS("YCbCr 4:2:0 Unsupported\n");
9613                 }
9614         }
9615
9616         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9617         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
9618                 power_domain_mask |= BIT_ULL(power_domain);
9619                 if (INTEL_GEN(dev_priv) >= 9)
9620                         skylake_get_pfit_config(crtc, pipe_config);
9621                 else
9622                         ironlake_get_pfit_config(crtc, pipe_config);
9623         }
9624
9625         if (hsw_crtc_supports_ips(crtc)) {
9626                 if (IS_HASWELL(dev_priv))
9627                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
9628                 else {
9629                         /*
9630                          * We cannot readout IPS state on broadwell, set to
9631                          * true so we can set it to a defined state on first
9632                          * commit.
9633                          */
9634                         pipe_config->ips_enabled = true;
9635                 }
9636         }
9637
9638         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
9639             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9640                 pipe_config->pixel_multiplier =
9641                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9642         } else {
9643                 pipe_config->pixel_multiplier = 1;
9644         }
9645
9646 out:
9647         for_each_power_domain(power_domain, power_domain_mask)
9648                 intel_display_power_put(dev_priv, power_domain);
9649
9650         return active;
9651 }
9652
9653 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
9654 {
9655         struct drm_i915_private *dev_priv =
9656                 to_i915(plane_state->base.plane->dev);
9657         const struct drm_framebuffer *fb = plane_state->base.fb;
9658         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
9659         u32 base;
9660
9661         if (INTEL_INFO(dev_priv)->cursor_needs_physical)
9662                 base = obj->phys_handle->busaddr;
9663         else
9664                 base = intel_plane_ggtt_offset(plane_state);
9665
9666         base += plane_state->color_plane[0].offset;
9667
9668         /* ILK+ do this automagically */
9669         if (HAS_GMCH_DISPLAY(dev_priv) &&
9670             plane_state->base.rotation & DRM_MODE_ROTATE_180)
9671                 base += (plane_state->base.crtc_h *
9672                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
9673
9674         return base;
9675 }
9676
9677 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
9678 {
9679         int x = plane_state->base.crtc_x;
9680         int y = plane_state->base.crtc_y;
9681         u32 pos = 0;
9682
9683         if (x < 0) {
9684                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
9685                 x = -x;
9686         }
9687         pos |= x << CURSOR_X_SHIFT;
9688
9689         if (y < 0) {
9690                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
9691                 y = -y;
9692         }
9693         pos |= y << CURSOR_Y_SHIFT;
9694
9695         return pos;
9696 }
9697
9698 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
9699 {
9700         const struct drm_mode_config *config =
9701                 &plane_state->base.plane->dev->mode_config;
9702         int width = plane_state->base.crtc_w;
9703         int height = plane_state->base.crtc_h;
9704
9705         return width > 0 && width <= config->cursor_width &&
9706                 height > 0 && height <= config->cursor_height;
9707 }
9708
9709 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
9710 {
9711         const struct drm_framebuffer *fb = plane_state->base.fb;
9712         unsigned int rotation = plane_state->base.rotation;
9713         int src_x, src_y;
9714         u32 offset;
9715         int ret;
9716
9717         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
9718         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
9719
9720         ret = intel_plane_check_stride(plane_state);
9721         if (ret)
9722                 return ret;
9723
9724         src_x = plane_state->base.src_x >> 16;
9725         src_y = plane_state->base.src_y >> 16;
9726
9727         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
9728         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
9729                                                     plane_state, 0);
9730
9731         if (src_x != 0 || src_y != 0) {
9732                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
9733                 return -EINVAL;
9734         }
9735
9736         plane_state->color_plane[0].offset = offset;
9737
9738         return 0;
9739 }
9740
9741 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
9742                               struct intel_plane_state *plane_state)
9743 {
9744         const struct drm_framebuffer *fb = plane_state->base.fb;
9745         int ret;
9746
9747         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
9748                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
9749                 return -EINVAL;
9750         }
9751
9752         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
9753                                                   &crtc_state->base,
9754                                                   DRM_PLANE_HELPER_NO_SCALING,
9755                                                   DRM_PLANE_HELPER_NO_SCALING,
9756                                                   true, true);
9757         if (ret)
9758                 return ret;
9759
9760         if (!plane_state->base.visible)
9761                 return 0;
9762
9763         ret = intel_plane_check_src_coordinates(plane_state);
9764         if (ret)
9765                 return ret;
9766
9767         ret = intel_cursor_check_surface(plane_state);
9768         if (ret)
9769                 return ret;
9770
9771         return 0;
9772 }
9773
9774 static unsigned int
9775 i845_cursor_max_stride(struct intel_plane *plane,
9776                        u32 pixel_format, u64 modifier,
9777                        unsigned int rotation)
9778 {
9779         return 2048;
9780 }
9781
9782 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
9783                            const struct intel_plane_state *plane_state)
9784 {
9785         return CURSOR_ENABLE |
9786                 CURSOR_GAMMA_ENABLE |
9787                 CURSOR_FORMAT_ARGB |
9788                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
9789 }
9790
9791 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
9792 {
9793         int width = plane_state->base.crtc_w;
9794
9795         /*
9796          * 845g/865g are only limited by the width of their cursors,
9797          * the height is arbitrary up to the precision of the register.
9798          */
9799         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
9800 }
9801
9802 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
9803                              struct intel_plane_state *plane_state)
9804 {
9805         const struct drm_framebuffer *fb = plane_state->base.fb;
9806         int ret;
9807
9808         ret = intel_check_cursor(crtc_state, plane_state);
9809         if (ret)
9810                 return ret;
9811
9812         /* if we want to turn off the cursor ignore width and height */
9813         if (!fb)
9814                 return 0;
9815
9816         /* Check for which cursor types we support */
9817         if (!i845_cursor_size_ok(plane_state)) {
9818                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
9819                           plane_state->base.crtc_w,
9820                           plane_state->base.crtc_h);
9821                 return -EINVAL;
9822         }
9823
9824         WARN_ON(plane_state->base.visible &&
9825                 plane_state->color_plane[0].stride != fb->pitches[0]);
9826
9827         switch (fb->pitches[0]) {
9828         case 256:
9829         case 512:
9830         case 1024:
9831         case 2048:
9832                 break;
9833         default:
9834                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
9835                               fb->pitches[0]);
9836                 return -EINVAL;
9837         }
9838
9839         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
9840
9841         return 0;
9842 }
9843
9844 static void i845_update_cursor(struct intel_plane *plane,
9845                                const struct intel_crtc_state *crtc_state,
9846                                const struct intel_plane_state *plane_state)
9847 {
9848         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9849         u32 cntl = 0, base = 0, pos = 0, size = 0;
9850         unsigned long irqflags;
9851
9852         if (plane_state && plane_state->base.visible) {
9853                 unsigned int width = plane_state->base.crtc_w;
9854                 unsigned int height = plane_state->base.crtc_h;
9855
9856                 cntl = plane_state->ctl;
9857                 size = (height << 12) | width;
9858
9859                 base = intel_cursor_base(plane_state);
9860                 pos = intel_cursor_position(plane_state);
9861         }
9862
9863         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
9864
9865         /* On these chipsets we can only modify the base/size/stride
9866          * whilst the cursor is disabled.
9867          */
9868         if (plane->cursor.base != base ||
9869             plane->cursor.size != size ||
9870             plane->cursor.cntl != cntl) {
9871                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
9872                 I915_WRITE_FW(CURBASE(PIPE_A), base);
9873                 I915_WRITE_FW(CURSIZE, size);
9874                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9875                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
9876
9877                 plane->cursor.base = base;
9878                 plane->cursor.size = size;
9879                 plane->cursor.cntl = cntl;
9880         } else {
9881                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9882         }
9883
9884         POSTING_READ_FW(CURCNTR(PIPE_A));
9885
9886         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
9887 }
9888
9889 static void i845_disable_cursor(struct intel_plane *plane,
9890                                 struct intel_crtc *crtc)
9891 {
9892         i845_update_cursor(plane, NULL, NULL);
9893 }
9894
9895 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
9896                                      enum pipe *pipe)
9897 {
9898         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9899         enum intel_display_power_domain power_domain;
9900         bool ret;
9901
9902         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
9903         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9904                 return false;
9905
9906         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
9907
9908         *pipe = PIPE_A;
9909
9910         intel_display_power_put(dev_priv, power_domain);
9911
9912         return ret;
9913 }
9914
9915 static unsigned int
9916 i9xx_cursor_max_stride(struct intel_plane *plane,
9917                        u32 pixel_format, u64 modifier,
9918                        unsigned int rotation)
9919 {
9920         return plane->base.dev->mode_config.cursor_width * 4;
9921 }
9922
9923 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
9924                            const struct intel_plane_state *plane_state)
9925 {
9926         struct drm_i915_private *dev_priv =
9927                 to_i915(plane_state->base.plane->dev);
9928         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9929         u32 cntl = 0;
9930
9931         if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
9932                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
9933
9934         if (INTEL_GEN(dev_priv) <= 10) {
9935                 cntl |= MCURSOR_GAMMA_ENABLE;
9936
9937                 if (HAS_DDI(dev_priv))
9938                         cntl |= MCURSOR_PIPE_CSC_ENABLE;
9939         }
9940
9941         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
9942                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
9943
9944         switch (plane_state->base.crtc_w) {
9945         case 64:
9946                 cntl |= MCURSOR_MODE_64_ARGB_AX;
9947                 break;
9948         case 128:
9949                 cntl |= MCURSOR_MODE_128_ARGB_AX;
9950                 break;
9951         case 256:
9952                 cntl |= MCURSOR_MODE_256_ARGB_AX;
9953                 break;
9954         default:
9955                 MISSING_CASE(plane_state->base.crtc_w);
9956                 return 0;
9957         }
9958
9959         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
9960                 cntl |= MCURSOR_ROTATE_180;
9961
9962         return cntl;
9963 }
9964
9965 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
9966 {
9967         struct drm_i915_private *dev_priv =
9968                 to_i915(plane_state->base.plane->dev);
9969         int width = plane_state->base.crtc_w;
9970         int height = plane_state->base.crtc_h;
9971
9972         if (!intel_cursor_size_ok(plane_state))
9973                 return false;
9974
9975         /* Cursor width is limited to a few power-of-two sizes */
9976         switch (width) {
9977         case 256:
9978         case 128:
9979         case 64:
9980                 break;
9981         default:
9982                 return false;
9983         }
9984
9985         /*
9986          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
9987          * height from 8 lines up to the cursor width, when the
9988          * cursor is not rotated. Everything else requires square
9989          * cursors.
9990          */
9991         if (HAS_CUR_FBC(dev_priv) &&
9992             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
9993                 if (height < 8 || height > width)
9994                         return false;
9995         } else {
9996                 if (height != width)
9997                         return false;
9998         }
9999
10000         return true;
10001 }
10002
10003 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10004                              struct intel_plane_state *plane_state)
10005 {
10006         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10007         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10008         const struct drm_framebuffer *fb = plane_state->base.fb;
10009         enum pipe pipe = plane->pipe;
10010         int ret;
10011
10012         ret = intel_check_cursor(crtc_state, plane_state);
10013         if (ret)
10014                 return ret;
10015
10016         /* if we want to turn off the cursor ignore width and height */
10017         if (!fb)
10018                 return 0;
10019
10020         /* Check for which cursor types we support */
10021         if (!i9xx_cursor_size_ok(plane_state)) {
10022                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10023                           plane_state->base.crtc_w,
10024                           plane_state->base.crtc_h);
10025                 return -EINVAL;
10026         }
10027
10028         WARN_ON(plane_state->base.visible &&
10029                 plane_state->color_plane[0].stride != fb->pitches[0]);
10030
10031         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10032                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10033                               fb->pitches[0], plane_state->base.crtc_w);
10034                 return -EINVAL;
10035         }
10036
10037         /*
10038          * There's something wrong with the cursor on CHV pipe C.
10039          * If it straddles the left edge of the screen then
10040          * moving it away from the edge or disabling it often
10041          * results in a pipe underrun, and often that can lead to
10042          * dead pipe (constant underrun reported, and it scans
10043          * out just a solid color). To recover from that, the
10044          * display power well must be turned off and on again.
10045          * Refuse the put the cursor into that compromised position.
10046          */
10047         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10048             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10049                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10050                 return -EINVAL;
10051         }
10052
10053         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10054
10055         return 0;
10056 }
10057
10058 static void i9xx_update_cursor(struct intel_plane *plane,
10059                                const struct intel_crtc_state *crtc_state,
10060                                const struct intel_plane_state *plane_state)
10061 {
10062         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10063         enum pipe pipe = plane->pipe;
10064         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10065         unsigned long irqflags;
10066
10067         if (plane_state && plane_state->base.visible) {
10068                 cntl = plane_state->ctl;
10069
10070                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10071                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10072
10073                 base = intel_cursor_base(plane_state);
10074                 pos = intel_cursor_position(plane_state);
10075         }
10076
10077         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10078
10079         /*
10080          * On some platforms writing CURCNTR first will also
10081          * cause CURPOS to be armed by the CURBASE write.
10082          * Without the CURCNTR write the CURPOS write would
10083          * arm itself. Thus we always start the full update
10084          * with a CURCNTR write.
10085          *
10086          * On other platforms CURPOS always requires the
10087          * CURBASE write to arm the update. Additonally
10088          * a write to any of the cursor register will cancel
10089          * an already armed cursor update. Thus leaving out
10090          * the CURBASE write after CURPOS could lead to a
10091          * cursor that doesn't appear to move, or even change
10092          * shape. Thus we always write CURBASE.
10093          *
10094          * CURCNTR and CUR_FBC_CTL are always
10095          * armed by the CURBASE write only.
10096          */
10097         if (plane->cursor.base != base ||
10098             plane->cursor.size != fbc_ctl ||
10099             plane->cursor.cntl != cntl) {
10100                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10101                 if (HAS_CUR_FBC(dev_priv))
10102                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10103                 I915_WRITE_FW(CURPOS(pipe), pos);
10104                 I915_WRITE_FW(CURBASE(pipe), base);
10105
10106                 plane->cursor.base = base;
10107                 plane->cursor.size = fbc_ctl;
10108                 plane->cursor.cntl = cntl;
10109         } else {
10110                 I915_WRITE_FW(CURPOS(pipe), pos);
10111                 I915_WRITE_FW(CURBASE(pipe), base);
10112         }
10113
10114         POSTING_READ_FW(CURBASE(pipe));
10115
10116         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10117 }
10118
10119 static void i9xx_disable_cursor(struct intel_plane *plane,
10120                                 struct intel_crtc *crtc)
10121 {
10122         i9xx_update_cursor(plane, NULL, NULL);
10123 }
10124
10125 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10126                                      enum pipe *pipe)
10127 {
10128         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10129         enum intel_display_power_domain power_domain;
10130         bool ret;
10131         u32 val;
10132
10133         /*
10134          * Not 100% correct for planes that can move between pipes,
10135          * but that's only the case for gen2-3 which don't have any
10136          * display power wells.
10137          */
10138         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10139         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10140                 return false;
10141
10142         val = I915_READ(CURCNTR(plane->pipe));
10143
10144         ret = val & MCURSOR_MODE;
10145
10146         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
10147                 *pipe = plane->pipe;
10148         else
10149                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
10150                         MCURSOR_PIPE_SELECT_SHIFT;
10151
10152         intel_display_power_put(dev_priv, power_domain);
10153
10154         return ret;
10155 }
10156
10157 /* VESA 640x480x72Hz mode to set on the pipe */
10158 static const struct drm_display_mode load_detect_mode = {
10159         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10160                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10161 };
10162
10163 struct drm_framebuffer *
10164 intel_framebuffer_create(struct drm_i915_gem_object *obj,
10165                          struct drm_mode_fb_cmd2 *mode_cmd)
10166 {
10167         struct intel_framebuffer *intel_fb;
10168         int ret;
10169
10170         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10171         if (!intel_fb)
10172                 return ERR_PTR(-ENOMEM);
10173
10174         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
10175         if (ret)
10176                 goto err;
10177
10178         return &intel_fb->base;
10179
10180 err:
10181         kfree(intel_fb);
10182         return ERR_PTR(ret);
10183 }
10184
10185 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
10186                                         struct drm_crtc *crtc)
10187 {
10188         struct drm_plane *plane;
10189         struct drm_plane_state *plane_state;
10190         int ret, i;
10191
10192         ret = drm_atomic_add_affected_planes(state, crtc);
10193         if (ret)
10194                 return ret;
10195
10196         for_each_new_plane_in_state(state, plane, plane_state, i) {
10197                 if (plane_state->crtc != crtc)
10198                         continue;
10199
10200                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
10201                 if (ret)
10202                         return ret;
10203
10204                 drm_atomic_set_fb_for_plane(plane_state, NULL);
10205         }
10206
10207         return 0;
10208 }
10209
10210 int intel_get_load_detect_pipe(struct drm_connector *connector,
10211                                const struct drm_display_mode *mode,
10212                                struct intel_load_detect_pipe *old,
10213                                struct drm_modeset_acquire_ctx *ctx)
10214 {
10215         struct intel_crtc *intel_crtc;
10216         struct intel_encoder *intel_encoder =
10217                 intel_attached_encoder(connector);
10218         struct drm_crtc *possible_crtc;
10219         struct drm_encoder *encoder = &intel_encoder->base;
10220         struct drm_crtc *crtc = NULL;
10221         struct drm_device *dev = encoder->dev;
10222         struct drm_i915_private *dev_priv = to_i915(dev);
10223         struct drm_mode_config *config = &dev->mode_config;
10224         struct drm_atomic_state *state = NULL, *restore_state = NULL;
10225         struct drm_connector_state *connector_state;
10226         struct intel_crtc_state *crtc_state;
10227         int ret, i = -1;
10228
10229         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10230                       connector->base.id, connector->name,
10231                       encoder->base.id, encoder->name);
10232
10233         old->restore_state = NULL;
10234
10235         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
10236
10237         /*
10238          * Algorithm gets a little messy:
10239          *
10240          *   - if the connector already has an assigned crtc, use it (but make
10241          *     sure it's on first)
10242          *
10243          *   - try to find the first unused crtc that can drive this connector,
10244          *     and use that if we find one
10245          */
10246
10247         /* See if we already have a CRTC for this connector */
10248         if (connector->state->crtc) {
10249                 crtc = connector->state->crtc;
10250
10251                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10252                 if (ret)
10253                         goto fail;
10254
10255                 /* Make sure the crtc and connector are running */
10256                 goto found;
10257         }
10258
10259         /* Find an unused one (if possible) */
10260         for_each_crtc(dev, possible_crtc) {
10261                 i++;
10262                 if (!(encoder->possible_crtcs & (1 << i)))
10263                         continue;
10264
10265                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
10266                 if (ret)
10267                         goto fail;
10268
10269                 if (possible_crtc->state->enable) {
10270                         drm_modeset_unlock(&possible_crtc->mutex);
10271                         continue;
10272                 }
10273
10274                 crtc = possible_crtc;
10275                 break;
10276         }
10277
10278         /*
10279          * If we didn't find an unused CRTC, don't use any.
10280          */
10281         if (!crtc) {
10282                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10283                 ret = -ENODEV;
10284                 goto fail;
10285         }
10286
10287 found:
10288         intel_crtc = to_intel_crtc(crtc);
10289
10290         state = drm_atomic_state_alloc(dev);
10291         restore_state = drm_atomic_state_alloc(dev);
10292         if (!state || !restore_state) {
10293                 ret = -ENOMEM;
10294                 goto fail;
10295         }
10296
10297         state->acquire_ctx = ctx;
10298         restore_state->acquire_ctx = ctx;
10299
10300         connector_state = drm_atomic_get_connector_state(state, connector);
10301         if (IS_ERR(connector_state)) {
10302                 ret = PTR_ERR(connector_state);
10303                 goto fail;
10304         }
10305
10306         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
10307         if (ret)
10308                 goto fail;
10309
10310         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10311         if (IS_ERR(crtc_state)) {
10312                 ret = PTR_ERR(crtc_state);
10313                 goto fail;
10314         }
10315
10316         crtc_state->base.active = crtc_state->base.enable = true;
10317
10318         if (!mode)
10319                 mode = &load_detect_mode;
10320
10321         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
10322         if (ret)
10323                 goto fail;
10324
10325         ret = intel_modeset_disable_planes(state, crtc);
10326         if (ret)
10327                 goto fail;
10328
10329         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
10330         if (!ret)
10331                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
10332         if (!ret)
10333                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
10334         if (ret) {
10335                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
10336                 goto fail;
10337         }
10338
10339         ret = drm_atomic_commit(state);
10340         if (ret) {
10341                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10342                 goto fail;
10343         }
10344
10345         old->restore_state = restore_state;
10346         drm_atomic_state_put(state);
10347
10348         /* let the connector get through one full cycle before testing */
10349         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
10350         return true;
10351
10352 fail:
10353         if (state) {
10354                 drm_atomic_state_put(state);
10355                 state = NULL;
10356         }
10357         if (restore_state) {
10358                 drm_atomic_state_put(restore_state);
10359                 restore_state = NULL;
10360         }
10361
10362         if (ret == -EDEADLK)
10363                 return ret;
10364
10365         return false;
10366 }
10367
10368 void intel_release_load_detect_pipe(struct drm_connector *connector,
10369                                     struct intel_load_detect_pipe *old,
10370                                     struct drm_modeset_acquire_ctx *ctx)
10371 {
10372         struct intel_encoder *intel_encoder =
10373                 intel_attached_encoder(connector);
10374         struct drm_encoder *encoder = &intel_encoder->base;
10375         struct drm_atomic_state *state = old->restore_state;
10376         int ret;
10377
10378         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10379                       connector->base.id, connector->name,
10380                       encoder->base.id, encoder->name);
10381
10382         if (!state)
10383                 return;
10384
10385         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
10386         if (ret)
10387                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
10388         drm_atomic_state_put(state);
10389 }
10390
10391 static int i9xx_pll_refclk(struct drm_device *dev,
10392                            const struct intel_crtc_state *pipe_config)
10393 {
10394         struct drm_i915_private *dev_priv = to_i915(dev);
10395         u32 dpll = pipe_config->dpll_hw_state.dpll;
10396
10397         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10398                 return dev_priv->vbt.lvds_ssc_freq;
10399         else if (HAS_PCH_SPLIT(dev_priv))
10400                 return 120000;
10401         else if (!IS_GEN2(dev_priv))
10402                 return 96000;
10403         else
10404                 return 48000;
10405 }
10406
10407 /* Returns the clock of the currently programmed mode of the given pipe. */
10408 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10409                                 struct intel_crtc_state *pipe_config)
10410 {
10411         struct drm_device *dev = crtc->base.dev;
10412         struct drm_i915_private *dev_priv = to_i915(dev);
10413         int pipe = pipe_config->cpu_transcoder;
10414         u32 dpll = pipe_config->dpll_hw_state.dpll;
10415         u32 fp;
10416         struct dpll clock;
10417         int port_clock;
10418         int refclk = i9xx_pll_refclk(dev, pipe_config);
10419
10420         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10421                 fp = pipe_config->dpll_hw_state.fp0;
10422         else
10423                 fp = pipe_config->dpll_hw_state.fp1;
10424
10425         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10426         if (IS_PINEVIEW(dev_priv)) {
10427                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10428                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10429         } else {
10430                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10431                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10432         }
10433
10434         if (!IS_GEN2(dev_priv)) {
10435                 if (IS_PINEVIEW(dev_priv))
10436                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10437                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10438                 else
10439                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10440                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10441
10442                 switch (dpll & DPLL_MODE_MASK) {
10443                 case DPLLB_MODE_DAC_SERIAL:
10444                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10445                                 5 : 10;
10446                         break;
10447                 case DPLLB_MODE_LVDS:
10448                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10449                                 7 : 14;
10450                         break;
10451                 default:
10452                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10453                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10454                         return;
10455                 }
10456
10457                 if (IS_PINEVIEW(dev_priv))
10458                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10459                 else
10460                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10461         } else {
10462                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
10463                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10464
10465                 if (is_lvds) {
10466                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10467                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10468
10469                         if (lvds & LVDS_CLKB_POWER_UP)
10470                                 clock.p2 = 7;
10471                         else
10472                                 clock.p2 = 14;
10473                 } else {
10474                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10475                                 clock.p1 = 2;
10476                         else {
10477                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10478                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10479                         }
10480                         if (dpll & PLL_P2_DIVIDE_BY_4)
10481                                 clock.p2 = 4;
10482                         else
10483                                 clock.p2 = 2;
10484                 }
10485
10486                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10487         }
10488
10489         /*
10490          * This value includes pixel_multiplier. We will use
10491          * port_clock to compute adjusted_mode.crtc_clock in the
10492          * encoder's get_config() function.
10493          */
10494         pipe_config->port_clock = port_clock;
10495 }
10496
10497 int intel_dotclock_calculate(int link_freq,
10498                              const struct intel_link_m_n *m_n)
10499 {
10500         /*
10501          * The calculation for the data clock is:
10502          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10503          * But we want to avoid losing precison if possible, so:
10504          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10505          *
10506          * and the link clock is simpler:
10507          * link_clock = (m * link_clock) / n
10508          */
10509
10510         if (!m_n->link_n)
10511                 return 0;
10512
10513         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
10514 }
10515
10516 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10517                                    struct intel_crtc_state *pipe_config)
10518 {
10519         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10520
10521         /* read out port_clock from the DPLL */
10522         i9xx_crtc_clock_get(crtc, pipe_config);
10523
10524         /*
10525          * In case there is an active pipe without active ports,
10526          * we may need some idea for the dotclock anyway.
10527          * Calculate one based on the FDI configuration.
10528          */
10529         pipe_config->base.adjusted_mode.crtc_clock =
10530                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
10531                                          &pipe_config->fdi_m_n);
10532 }
10533
10534 /* Returns the currently programmed mode of the given encoder. */
10535 struct drm_display_mode *
10536 intel_encoder_current_mode(struct intel_encoder *encoder)
10537 {
10538         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
10539         struct intel_crtc_state *crtc_state;
10540         struct drm_display_mode *mode;
10541         struct intel_crtc *crtc;
10542         enum pipe pipe;
10543
10544         if (!encoder->get_hw_state(encoder, &pipe))
10545                 return NULL;
10546
10547         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
10548
10549         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10550         if (!mode)
10551                 return NULL;
10552
10553         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
10554         if (!crtc_state) {
10555                 kfree(mode);
10556                 return NULL;
10557         }
10558
10559         crtc_state->base.crtc = &crtc->base;
10560
10561         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
10562                 kfree(crtc_state);
10563                 kfree(mode);
10564                 return NULL;
10565         }
10566
10567         encoder->get_config(encoder, crtc_state);
10568
10569         intel_mode_from_pipe_config(mode, crtc_state);
10570
10571         kfree(crtc_state);
10572
10573         return mode;
10574 }
10575
10576 static void intel_crtc_destroy(struct drm_crtc *crtc)
10577 {
10578         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10579
10580         drm_crtc_cleanup(crtc);
10581         kfree(intel_crtc);
10582 }
10583
10584 /**
10585  * intel_wm_need_update - Check whether watermarks need updating
10586  * @plane: drm plane
10587  * @state: new plane state
10588  *
10589  * Check current plane state versus the new one to determine whether
10590  * watermarks need to be recalculated.
10591  *
10592  * Returns true or false.
10593  */
10594 static bool intel_wm_need_update(struct drm_plane *plane,
10595                                  struct drm_plane_state *state)
10596 {
10597         struct intel_plane_state *new = to_intel_plane_state(state);
10598         struct intel_plane_state *cur = to_intel_plane_state(plane->state);
10599
10600         /* Update watermarks on tiling or size changes. */
10601         if (new->base.visible != cur->base.visible)
10602                 return true;
10603
10604         if (!cur->base.fb || !new->base.fb)
10605                 return false;
10606
10607         if (cur->base.fb->modifier != new->base.fb->modifier ||
10608             cur->base.rotation != new->base.rotation ||
10609             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
10610             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
10611             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
10612             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
10613                 return true;
10614
10615         return false;
10616 }
10617
10618 static bool needs_scaling(const struct intel_plane_state *state)
10619 {
10620         int src_w = drm_rect_width(&state->base.src) >> 16;
10621         int src_h = drm_rect_height(&state->base.src) >> 16;
10622         int dst_w = drm_rect_width(&state->base.dst);
10623         int dst_h = drm_rect_height(&state->base.dst);
10624
10625         return (src_w != dst_w || src_h != dst_h);
10626 }
10627
10628 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
10629                                     struct drm_crtc_state *crtc_state,
10630                                     const struct intel_plane_state *old_plane_state,
10631                                     struct drm_plane_state *plane_state)
10632 {
10633         struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
10634         struct drm_crtc *crtc = crtc_state->crtc;
10635         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10636         struct intel_plane *plane = to_intel_plane(plane_state->plane);
10637         struct drm_device *dev = crtc->dev;
10638         struct drm_i915_private *dev_priv = to_i915(dev);
10639         bool mode_changed = needs_modeset(crtc_state);
10640         bool was_crtc_enabled = old_crtc_state->base.active;
10641         bool is_crtc_enabled = crtc_state->active;
10642         bool turn_off, turn_on, visible, was_visible;
10643         struct drm_framebuffer *fb = plane_state->fb;
10644         int ret;
10645
10646         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
10647                 ret = skl_update_scaler_plane(
10648                         to_intel_crtc_state(crtc_state),
10649                         to_intel_plane_state(plane_state));
10650                 if (ret)
10651                         return ret;
10652         }
10653
10654         was_visible = old_plane_state->base.visible;
10655         visible = plane_state->visible;
10656
10657         if (!was_crtc_enabled && WARN_ON(was_visible))
10658                 was_visible = false;
10659
10660         /*
10661          * Visibility is calculated as if the crtc was on, but
10662          * after scaler setup everything depends on it being off
10663          * when the crtc isn't active.
10664          *
10665          * FIXME this is wrong for watermarks. Watermarks should also
10666          * be computed as if the pipe would be active. Perhaps move
10667          * per-plane wm computation to the .check_plane() hook, and
10668          * only combine the results from all planes in the current place?
10669          */
10670         if (!is_crtc_enabled) {
10671                 plane_state->visible = visible = false;
10672                 to_intel_crtc_state(crtc_state)->active_planes &= ~BIT(plane->id);
10673         }
10674
10675         if (!was_visible && !visible)
10676                 return 0;
10677
10678         if (fb != old_plane_state->base.fb)
10679                 pipe_config->fb_changed = true;
10680
10681         turn_off = was_visible && (!visible || mode_changed);
10682         turn_on = visible && (!was_visible || mode_changed);
10683
10684         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
10685                          intel_crtc->base.base.id, intel_crtc->base.name,
10686                          plane->base.base.id, plane->base.name,
10687                          fb ? fb->base.id : -1);
10688
10689         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
10690                          plane->base.base.id, plane->base.name,
10691                          was_visible, visible,
10692                          turn_off, turn_on, mode_changed);
10693
10694         if (turn_on) {
10695                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10696                         pipe_config->update_wm_pre = true;
10697
10698                 /* must disable cxsr around plane enable/disable */
10699                 if (plane->id != PLANE_CURSOR)
10700                         pipe_config->disable_cxsr = true;
10701         } else if (turn_off) {
10702                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10703                         pipe_config->update_wm_post = true;
10704
10705                 /* must disable cxsr around plane enable/disable */
10706                 if (plane->id != PLANE_CURSOR)
10707                         pipe_config->disable_cxsr = true;
10708         } else if (intel_wm_need_update(&plane->base, plane_state)) {
10709                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
10710                         /* FIXME bollocks */
10711                         pipe_config->update_wm_pre = true;
10712                         pipe_config->update_wm_post = true;
10713                 }
10714         }
10715
10716         if (visible || was_visible)
10717                 pipe_config->fb_bits |= plane->frontbuffer_bit;
10718
10719         /*
10720          * WaCxSRDisabledForSpriteScaling:ivb
10721          *
10722          * cstate->update_wm was already set above, so this flag will
10723          * take effect when we commit and program watermarks.
10724          */
10725         if (plane->id == PLANE_SPRITE0 && IS_IVYBRIDGE(dev_priv) &&
10726             needs_scaling(to_intel_plane_state(plane_state)) &&
10727             !needs_scaling(old_plane_state))
10728                 pipe_config->disable_lp_wm = true;
10729
10730         return 0;
10731 }
10732
10733 static bool encoders_cloneable(const struct intel_encoder *a,
10734                                const struct intel_encoder *b)
10735 {
10736         /* masks could be asymmetric, so check both ways */
10737         return a == b || (a->cloneable & (1 << b->type) &&
10738                           b->cloneable & (1 << a->type));
10739 }
10740
10741 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
10742                                          struct intel_crtc *crtc,
10743                                          struct intel_encoder *encoder)
10744 {
10745         struct intel_encoder *source_encoder;
10746         struct drm_connector *connector;
10747         struct drm_connector_state *connector_state;
10748         int i;
10749
10750         for_each_new_connector_in_state(state, connector, connector_state, i) {
10751                 if (connector_state->crtc != &crtc->base)
10752                         continue;
10753
10754                 source_encoder =
10755                         to_intel_encoder(connector_state->best_encoder);
10756                 if (!encoders_cloneable(encoder, source_encoder))
10757                         return false;
10758         }
10759
10760         return true;
10761 }
10762
10763 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
10764                                    struct drm_crtc_state *crtc_state)
10765 {
10766         struct drm_device *dev = crtc->dev;
10767         struct drm_i915_private *dev_priv = to_i915(dev);
10768         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10769         struct intel_crtc_state *pipe_config =
10770                 to_intel_crtc_state(crtc_state);
10771         struct drm_atomic_state *state = crtc_state->state;
10772         int ret;
10773         bool mode_changed = needs_modeset(crtc_state);
10774
10775         if (mode_changed && !crtc_state->active)
10776                 pipe_config->update_wm_post = true;
10777
10778         if (mode_changed && crtc_state->enable &&
10779             dev_priv->display.crtc_compute_clock &&
10780             !WARN_ON(pipe_config->shared_dpll)) {
10781                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
10782                                                            pipe_config);
10783                 if (ret)
10784                         return ret;
10785         }
10786
10787         if (crtc_state->color_mgmt_changed) {
10788                 ret = intel_color_check(crtc, crtc_state);
10789                 if (ret)
10790                         return ret;
10791
10792                 /*
10793                  * Changing color management on Intel hardware is
10794                  * handled as part of planes update.
10795                  */
10796                 crtc_state->planes_changed = true;
10797         }
10798
10799         ret = 0;
10800         if (dev_priv->display.compute_pipe_wm) {
10801                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
10802                 if (ret) {
10803                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
10804                         return ret;
10805                 }
10806         }
10807
10808         if (dev_priv->display.compute_intermediate_wm &&
10809             !to_intel_atomic_state(state)->skip_intermediate_wm) {
10810                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
10811                         return 0;
10812
10813                 /*
10814                  * Calculate 'intermediate' watermarks that satisfy both the
10815                  * old state and the new state.  We can program these
10816                  * immediately.
10817                  */
10818                 ret = dev_priv->display.compute_intermediate_wm(dev,
10819                                                                 intel_crtc,
10820                                                                 pipe_config);
10821                 if (ret) {
10822                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
10823                         return ret;
10824                 }
10825         } else if (dev_priv->display.compute_intermediate_wm) {
10826                 if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9)
10827                         pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
10828         }
10829
10830         if (INTEL_GEN(dev_priv) >= 9) {
10831                 if (mode_changed)
10832                         ret = skl_update_scaler_crtc(pipe_config);
10833
10834                 if (!ret)
10835                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
10836                                                             pipe_config);
10837                 if (!ret)
10838                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
10839                                                          pipe_config);
10840         }
10841
10842         if (HAS_IPS(dev_priv))
10843                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
10844
10845         return ret;
10846 }
10847
10848 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
10849         .atomic_check = intel_crtc_atomic_check,
10850 };
10851
10852 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
10853 {
10854         struct intel_connector *connector;
10855         struct drm_connector_list_iter conn_iter;
10856
10857         drm_connector_list_iter_begin(dev, &conn_iter);
10858         for_each_intel_connector_iter(connector, &conn_iter) {
10859                 if (connector->base.state->crtc)
10860                         drm_connector_put(&connector->base);
10861
10862                 if (connector->base.encoder) {
10863                         connector->base.state->best_encoder =
10864                                 connector->base.encoder;
10865                         connector->base.state->crtc =
10866                                 connector->base.encoder->crtc;
10867
10868                         drm_connector_get(&connector->base);
10869                 } else {
10870                         connector->base.state->best_encoder = NULL;
10871                         connector->base.state->crtc = NULL;
10872                 }
10873         }
10874         drm_connector_list_iter_end(&conn_iter);
10875 }
10876
10877 static void
10878 connected_sink_compute_bpp(struct intel_connector *connector,
10879                            struct intel_crtc_state *pipe_config)
10880 {
10881         const struct drm_display_info *info = &connector->base.display_info;
10882         int bpp = pipe_config->pipe_bpp;
10883
10884         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
10885                       connector->base.base.id,
10886                       connector->base.name);
10887
10888         /* Don't use an invalid EDID bpc value */
10889         if (info->bpc != 0 && info->bpc * 3 < bpp) {
10890                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
10891                               bpp, info->bpc * 3);
10892                 pipe_config->pipe_bpp = info->bpc * 3;
10893         }
10894
10895         /* Clamp bpp to 8 on screens without EDID 1.4 */
10896         if (info->bpc == 0 && bpp > 24) {
10897                 DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n",
10898                               bpp);
10899                 pipe_config->pipe_bpp = 24;
10900         }
10901 }
10902
10903 static int
10904 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
10905                           struct intel_crtc_state *pipe_config)
10906 {
10907         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10908         struct drm_atomic_state *state;
10909         struct drm_connector *connector;
10910         struct drm_connector_state *connector_state;
10911         int bpp, i;
10912
10913         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
10914             IS_CHERRYVIEW(dev_priv)))
10915                 bpp = 10*3;
10916         else if (INTEL_GEN(dev_priv) >= 5)
10917                 bpp = 12*3;
10918         else
10919                 bpp = 8*3;
10920
10921
10922         pipe_config->pipe_bpp = bpp;
10923
10924         state = pipe_config->base.state;
10925
10926         /* Clamp display bpp to EDID value */
10927         for_each_new_connector_in_state(state, connector, connector_state, i) {
10928                 if (connector_state->crtc != &crtc->base)
10929                         continue;
10930
10931                 connected_sink_compute_bpp(to_intel_connector(connector),
10932                                            pipe_config);
10933         }
10934
10935         return bpp;
10936 }
10937
10938 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
10939 {
10940         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
10941                         "type: 0x%x flags: 0x%x\n",
10942                 mode->crtc_clock,
10943                 mode->crtc_hdisplay, mode->crtc_hsync_start,
10944                 mode->crtc_hsync_end, mode->crtc_htotal,
10945                 mode->crtc_vdisplay, mode->crtc_vsync_start,
10946                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
10947 }
10948
10949 static inline void
10950 intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
10951                       unsigned int lane_count, struct intel_link_m_n *m_n)
10952 {
10953         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
10954                       id, lane_count,
10955                       m_n->gmch_m, m_n->gmch_n,
10956                       m_n->link_m, m_n->link_n, m_n->tu);
10957 }
10958
10959 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
10960
10961 static const char * const output_type_str[] = {
10962         OUTPUT_TYPE(UNUSED),
10963         OUTPUT_TYPE(ANALOG),
10964         OUTPUT_TYPE(DVO),
10965         OUTPUT_TYPE(SDVO),
10966         OUTPUT_TYPE(LVDS),
10967         OUTPUT_TYPE(TVOUT),
10968         OUTPUT_TYPE(HDMI),
10969         OUTPUT_TYPE(DP),
10970         OUTPUT_TYPE(EDP),
10971         OUTPUT_TYPE(DSI),
10972         OUTPUT_TYPE(DDI),
10973         OUTPUT_TYPE(DP_MST),
10974 };
10975
10976 #undef OUTPUT_TYPE
10977
10978 static void snprintf_output_types(char *buf, size_t len,
10979                                   unsigned int output_types)
10980 {
10981         char *str = buf;
10982         int i;
10983
10984         str[0] = '\0';
10985
10986         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
10987                 int r;
10988
10989                 if ((output_types & BIT(i)) == 0)
10990                         continue;
10991
10992                 r = snprintf(str, len, "%s%s",
10993                              str != buf ? "," : "", output_type_str[i]);
10994                 if (r >= len)
10995                         break;
10996                 str += r;
10997                 len -= r;
10998
10999                 output_types &= ~BIT(i);
11000         }
11001
11002         WARN_ON_ONCE(output_types != 0);
11003 }
11004
11005 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11006                                    struct intel_crtc_state *pipe_config,
11007                                    const char *context)
11008 {
11009         struct drm_device *dev = crtc->base.dev;
11010         struct drm_i915_private *dev_priv = to_i915(dev);
11011         struct drm_plane *plane;
11012         struct intel_plane *intel_plane;
11013         struct intel_plane_state *state;
11014         struct drm_framebuffer *fb;
11015         char buf[64];
11016
11017         DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
11018                       crtc->base.base.id, crtc->base.name, context);
11019
11020         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
11021         DRM_DEBUG_KMS("output_types: %s (0x%x)\n",
11022                       buf, pipe_config->output_types);
11023
11024         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
11025                       transcoder_name(pipe_config->cpu_transcoder),
11026                       pipe_config->pipe_bpp, pipe_config->dither);
11027
11028         if (pipe_config->has_pch_encoder)
11029                 intel_dump_m_n_config(pipe_config, "fdi",
11030                                       pipe_config->fdi_lanes,
11031                                       &pipe_config->fdi_m_n);
11032
11033         if (pipe_config->ycbcr420)
11034                 DRM_DEBUG_KMS("YCbCr 4:2:0 output enabled\n");
11035
11036         if (intel_crtc_has_dp_encoder(pipe_config)) {
11037                 intel_dump_m_n_config(pipe_config, "dp m_n",
11038                                 pipe_config->lane_count, &pipe_config->dp_m_n);
11039                 if (pipe_config->has_drrs)
11040                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
11041                                               pipe_config->lane_count,
11042                                               &pipe_config->dp_m2_n2);
11043         }
11044
11045         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
11046                       pipe_config->has_audio, pipe_config->has_infoframe);
11047
11048         DRM_DEBUG_KMS("requested mode:\n");
11049         drm_mode_debug_printmodeline(&pipe_config->base.mode);
11050         DRM_DEBUG_KMS("adjusted mode:\n");
11051         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
11052         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
11053         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
11054                       pipe_config->port_clock,
11055                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
11056                       pipe_config->pixel_rate);
11057
11058         if (INTEL_GEN(dev_priv) >= 9)
11059                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
11060                               crtc->num_scalers,
11061                               pipe_config->scaler_state.scaler_users,
11062                               pipe_config->scaler_state.scaler_id);
11063
11064         if (HAS_GMCH_DISPLAY(dev_priv))
11065                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
11066                               pipe_config->gmch_pfit.control,
11067                               pipe_config->gmch_pfit.pgm_ratios,
11068                               pipe_config->gmch_pfit.lvds_border_bits);
11069         else
11070                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
11071                               pipe_config->pch_pfit.pos,
11072                               pipe_config->pch_pfit.size,
11073                               enableddisabled(pipe_config->pch_pfit.enabled));
11074
11075         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
11076                       pipe_config->ips_enabled, pipe_config->double_wide);
11077
11078         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
11079
11080         DRM_DEBUG_KMS("planes on this crtc\n");
11081         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
11082                 struct drm_format_name_buf format_name;
11083                 intel_plane = to_intel_plane(plane);
11084                 if (intel_plane->pipe != crtc->pipe)
11085                         continue;
11086
11087                 state = to_intel_plane_state(plane->state);
11088                 fb = state->base.fb;
11089                 if (!fb) {
11090                         DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
11091                                       plane->base.id, plane->name, state->scaler_id);
11092                         continue;
11093                 }
11094
11095                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
11096                               plane->base.id, plane->name,
11097                               fb->base.id, fb->width, fb->height,
11098                               drm_get_format_name(fb->format->format, &format_name));
11099                 if (INTEL_GEN(dev_priv) >= 9)
11100                         DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
11101                                       state->scaler_id,
11102                                       state->base.src.x1 >> 16,
11103                                       state->base.src.y1 >> 16,
11104                                       drm_rect_width(&state->base.src) >> 16,
11105                                       drm_rect_height(&state->base.src) >> 16,
11106                                       state->base.dst.x1, state->base.dst.y1,
11107                                       drm_rect_width(&state->base.dst),
11108                                       drm_rect_height(&state->base.dst));
11109         }
11110 }
11111
11112 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
11113 {
11114         struct drm_device *dev = state->dev;
11115         struct drm_connector *connector;
11116         struct drm_connector_list_iter conn_iter;
11117         unsigned int used_ports = 0;
11118         unsigned int used_mst_ports = 0;
11119         bool ret = true;
11120
11121         /*
11122          * Walk the connector list instead of the encoder
11123          * list to detect the problem on ddi platforms
11124          * where there's just one encoder per digital port.
11125          */
11126         drm_connector_list_iter_begin(dev, &conn_iter);
11127         drm_for_each_connector_iter(connector, &conn_iter) {
11128                 struct drm_connector_state *connector_state;
11129                 struct intel_encoder *encoder;
11130
11131                 connector_state = drm_atomic_get_new_connector_state(state, connector);
11132                 if (!connector_state)
11133                         connector_state = connector->state;
11134
11135                 if (!connector_state->best_encoder)
11136                         continue;
11137
11138                 encoder = to_intel_encoder(connector_state->best_encoder);
11139
11140                 WARN_ON(!connector_state->crtc);
11141
11142                 switch (encoder->type) {
11143                         unsigned int port_mask;
11144                 case INTEL_OUTPUT_DDI:
11145                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
11146                                 break;
11147                         /* else: fall through */
11148                 case INTEL_OUTPUT_DP:
11149                 case INTEL_OUTPUT_HDMI:
11150                 case INTEL_OUTPUT_EDP:
11151                         port_mask = 1 << encoder->port;
11152
11153                         /* the same port mustn't appear more than once */
11154                         if (used_ports & port_mask)
11155                                 ret = false;
11156
11157                         used_ports |= port_mask;
11158                         break;
11159                 case INTEL_OUTPUT_DP_MST:
11160                         used_mst_ports |=
11161                                 1 << encoder->port;
11162                         break;
11163                 default:
11164                         break;
11165                 }
11166         }
11167         drm_connector_list_iter_end(&conn_iter);
11168
11169         /* can't mix MST and SST/HDMI on the same port */
11170         if (used_ports & used_mst_ports)
11171                 return false;
11172
11173         return ret;
11174 }
11175
11176 static void
11177 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
11178 {
11179         struct drm_i915_private *dev_priv =
11180                 to_i915(crtc_state->base.crtc->dev);
11181         struct intel_crtc_scaler_state scaler_state;
11182         struct intel_dpll_hw_state dpll_hw_state;
11183         struct intel_shared_dpll *shared_dpll;
11184         struct intel_crtc_wm_state wm_state;
11185         bool force_thru, ips_force_disable;
11186
11187         /* FIXME: before the switch to atomic started, a new pipe_config was
11188          * kzalloc'd. Code that depends on any field being zero should be
11189          * fixed, so that the crtc_state can be safely duplicated. For now,
11190          * only fields that are know to not cause problems are preserved. */
11191
11192         scaler_state = crtc_state->scaler_state;
11193         shared_dpll = crtc_state->shared_dpll;
11194         dpll_hw_state = crtc_state->dpll_hw_state;
11195         force_thru = crtc_state->pch_pfit.force_thru;
11196         ips_force_disable = crtc_state->ips_force_disable;
11197         if (IS_G4X(dev_priv) ||
11198             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11199                 wm_state = crtc_state->wm;
11200
11201         /* Keep base drm_crtc_state intact, only clear our extended struct */
11202         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
11203         memset(&crtc_state->base + 1, 0,
11204                sizeof(*crtc_state) - sizeof(crtc_state->base));
11205
11206         crtc_state->scaler_state = scaler_state;
11207         crtc_state->shared_dpll = shared_dpll;
11208         crtc_state->dpll_hw_state = dpll_hw_state;
11209         crtc_state->pch_pfit.force_thru = force_thru;
11210         crtc_state->ips_force_disable = ips_force_disable;
11211         if (IS_G4X(dev_priv) ||
11212             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11213                 crtc_state->wm = wm_state;
11214 }
11215
11216 static int
11217 intel_modeset_pipe_config(struct drm_crtc *crtc,
11218                           struct intel_crtc_state *pipe_config)
11219 {
11220         struct drm_atomic_state *state = pipe_config->base.state;
11221         struct intel_encoder *encoder;
11222         struct drm_connector *connector;
11223         struct drm_connector_state *connector_state;
11224         int base_bpp, ret = -EINVAL;
11225         int i;
11226         bool retry = true;
11227
11228         clear_intel_crtc_state(pipe_config);
11229
11230         pipe_config->cpu_transcoder =
11231                 (enum transcoder) to_intel_crtc(crtc)->pipe;
11232
11233         /*
11234          * Sanitize sync polarity flags based on requested ones. If neither
11235          * positive or negative polarity is requested, treat this as meaning
11236          * negative polarity.
11237          */
11238         if (!(pipe_config->base.adjusted_mode.flags &
11239               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
11240                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
11241
11242         if (!(pipe_config->base.adjusted_mode.flags &
11243               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
11244                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
11245
11246         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
11247                                              pipe_config);
11248         if (base_bpp < 0)
11249                 goto fail;
11250
11251         /*
11252          * Determine the real pipe dimensions. Note that stereo modes can
11253          * increase the actual pipe size due to the frame doubling and
11254          * insertion of additional space for blanks between the frame. This
11255          * is stored in the crtc timings. We use the requested mode to do this
11256          * computation to clearly distinguish it from the adjusted mode, which
11257          * can be changed by the connectors in the below retry loop.
11258          */
11259         drm_mode_get_hv_timing(&pipe_config->base.mode,
11260                                &pipe_config->pipe_src_w,
11261                                &pipe_config->pipe_src_h);
11262
11263         for_each_new_connector_in_state(state, connector, connector_state, i) {
11264                 if (connector_state->crtc != crtc)
11265                         continue;
11266
11267                 encoder = to_intel_encoder(connector_state->best_encoder);
11268
11269                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
11270                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11271                         goto fail;
11272                 }
11273
11274                 /*
11275                  * Determine output_types before calling the .compute_config()
11276                  * hooks so that the hooks can use this information safely.
11277                  */
11278                 if (encoder->compute_output_type)
11279                         pipe_config->output_types |=
11280                                 BIT(encoder->compute_output_type(encoder, pipe_config,
11281                                                                  connector_state));
11282                 else
11283                         pipe_config->output_types |= BIT(encoder->type);
11284         }
11285
11286 encoder_retry:
11287         /* Ensure the port clock defaults are reset when retrying. */
11288         pipe_config->port_clock = 0;
11289         pipe_config->pixel_multiplier = 1;
11290
11291         /* Fill in default crtc timings, allow encoders to overwrite them. */
11292         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
11293                               CRTC_STEREO_DOUBLE);
11294
11295         /* Pass our mode to the connectors and the CRTC to give them a chance to
11296          * adjust it according to limitations or connector properties, and also
11297          * a chance to reject the mode entirely.
11298          */
11299         for_each_new_connector_in_state(state, connector, connector_state, i) {
11300                 if (connector_state->crtc != crtc)
11301                         continue;
11302
11303                 encoder = to_intel_encoder(connector_state->best_encoder);
11304
11305                 if (!(encoder->compute_config(encoder, pipe_config, connector_state))) {
11306                         DRM_DEBUG_KMS("Encoder config failure\n");
11307                         goto fail;
11308                 }
11309         }
11310
11311         /* Set default port clock if not overwritten by the encoder. Needs to be
11312          * done afterwards in case the encoder adjusts the mode. */
11313         if (!pipe_config->port_clock)
11314                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
11315                         * pipe_config->pixel_multiplier;
11316
11317         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
11318         if (ret < 0) {
11319                 DRM_DEBUG_KMS("CRTC fixup failed\n");
11320                 goto fail;
11321         }
11322
11323         if (ret == RETRY) {
11324                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
11325                         ret = -EINVAL;
11326                         goto fail;
11327                 }
11328
11329                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
11330                 retry = false;
11331                 goto encoder_retry;
11332         }
11333
11334         /* Dithering seems to not pass-through bits correctly when it should, so
11335          * only enable it on 6bpc panels and when its not a compliance
11336          * test requesting 6bpc video pattern.
11337          */
11338         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
11339                 !pipe_config->dither_force_disable;
11340         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
11341                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
11342
11343 fail:
11344         return ret;
11345 }
11346
11347 static bool intel_fuzzy_clock_check(int clock1, int clock2)
11348 {
11349         int diff;
11350
11351         if (clock1 == clock2)
11352                 return true;
11353
11354         if (!clock1 || !clock2)
11355                 return false;
11356
11357         diff = abs(clock1 - clock2);
11358
11359         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
11360                 return true;
11361
11362         return false;
11363 }
11364
11365 static bool
11366 intel_compare_m_n(unsigned int m, unsigned int n,
11367                   unsigned int m2, unsigned int n2,
11368                   bool exact)
11369 {
11370         if (m == m2 && n == n2)
11371                 return true;
11372
11373         if (exact || !m || !n || !m2 || !n2)
11374                 return false;
11375
11376         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
11377
11378         if (n > n2) {
11379                 while (n > n2) {
11380                         m2 <<= 1;
11381                         n2 <<= 1;
11382                 }
11383         } else if (n < n2) {
11384                 while (n < n2) {
11385                         m <<= 1;
11386                         n <<= 1;
11387                 }
11388         }
11389
11390         if (n != n2)
11391                 return false;
11392
11393         return intel_fuzzy_clock_check(m, m2);
11394 }
11395
11396 static bool
11397 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
11398                        struct intel_link_m_n *m2_n2,
11399                        bool adjust)
11400 {
11401         if (m_n->tu == m2_n2->tu &&
11402             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
11403                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
11404             intel_compare_m_n(m_n->link_m, m_n->link_n,
11405                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
11406                 if (adjust)
11407                         *m2_n2 = *m_n;
11408
11409                 return true;
11410         }
11411
11412         return false;
11413 }
11414
11415 static void __printf(3, 4)
11416 pipe_config_err(bool adjust, const char *name, const char *format, ...)
11417 {
11418         struct va_format vaf;
11419         va_list args;
11420
11421         va_start(args, format);
11422         vaf.fmt = format;
11423         vaf.va = &args;
11424
11425         if (adjust)
11426                 drm_dbg(DRM_UT_KMS, "mismatch in %s %pV", name, &vaf);
11427         else
11428                 drm_err("mismatch in %s %pV", name, &vaf);
11429
11430         va_end(args);
11431 }
11432
11433 static bool
11434 intel_pipe_config_compare(struct drm_i915_private *dev_priv,
11435                           struct intel_crtc_state *current_config,
11436                           struct intel_crtc_state *pipe_config,
11437                           bool adjust)
11438 {
11439         bool ret = true;
11440         bool fixup_inherited = adjust &&
11441                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
11442                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
11443
11444 #define PIPE_CONF_CHECK_X(name) do { \
11445         if (current_config->name != pipe_config->name) { \
11446                 pipe_config_err(adjust, __stringify(name), \
11447                           "(expected 0x%08x, found 0x%08x)\n", \
11448                           current_config->name, \
11449                           pipe_config->name); \
11450                 ret = false; \
11451         } \
11452 } while (0)
11453
11454 #define PIPE_CONF_CHECK_I(name) do { \
11455         if (current_config->name != pipe_config->name) { \
11456                 pipe_config_err(adjust, __stringify(name), \
11457                           "(expected %i, found %i)\n", \
11458                           current_config->name, \
11459                           pipe_config->name); \
11460                 ret = false; \
11461         } \
11462 } while (0)
11463
11464 #define PIPE_CONF_CHECK_BOOL(name) do { \
11465         if (current_config->name != pipe_config->name) { \
11466                 pipe_config_err(adjust, __stringify(name), \
11467                           "(expected %s, found %s)\n", \
11468                           yesno(current_config->name), \
11469                           yesno(pipe_config->name)); \
11470                 ret = false; \
11471         } \
11472 } while (0)
11473
11474 /*
11475  * Checks state where we only read out the enabling, but not the entire
11476  * state itself (like full infoframes or ELD for audio). These states
11477  * require a full modeset on bootup to fix up.
11478  */
11479 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
11480         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
11481                 PIPE_CONF_CHECK_BOOL(name); \
11482         } else { \
11483                 pipe_config_err(adjust, __stringify(name), \
11484                           "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
11485                           yesno(current_config->name), \
11486                           yesno(pipe_config->name)); \
11487                 ret = false; \
11488         } \
11489 } while (0)
11490
11491 #define PIPE_CONF_CHECK_P(name) do { \
11492         if (current_config->name != pipe_config->name) { \
11493                 pipe_config_err(adjust, __stringify(name), \
11494                           "(expected %p, found %p)\n", \
11495                           current_config->name, \
11496                           pipe_config->name); \
11497                 ret = false; \
11498         } \
11499 } while (0)
11500
11501 #define PIPE_CONF_CHECK_M_N(name) do { \
11502         if (!intel_compare_link_m_n(&current_config->name, \
11503                                     &pipe_config->name,\
11504                                     adjust)) { \
11505                 pipe_config_err(adjust, __stringify(name), \
11506                           "(expected tu %i gmch %i/%i link %i/%i, " \
11507                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11508                           current_config->name.tu, \
11509                           current_config->name.gmch_m, \
11510                           current_config->name.gmch_n, \
11511                           current_config->name.link_m, \
11512                           current_config->name.link_n, \
11513                           pipe_config->name.tu, \
11514                           pipe_config->name.gmch_m, \
11515                           pipe_config->name.gmch_n, \
11516                           pipe_config->name.link_m, \
11517                           pipe_config->name.link_n); \
11518                 ret = false; \
11519         } \
11520 } while (0)
11521
11522 /* This is required for BDW+ where there is only one set of registers for
11523  * switching between high and low RR.
11524  * This macro can be used whenever a comparison has to be made between one
11525  * hw state and multiple sw state variables.
11526  */
11527 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
11528         if (!intel_compare_link_m_n(&current_config->name, \
11529                                     &pipe_config->name, adjust) && \
11530             !intel_compare_link_m_n(&current_config->alt_name, \
11531                                     &pipe_config->name, adjust)) { \
11532                 pipe_config_err(adjust, __stringify(name), \
11533                           "(expected tu %i gmch %i/%i link %i/%i, " \
11534                           "or tu %i gmch %i/%i link %i/%i, " \
11535                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11536                           current_config->name.tu, \
11537                           current_config->name.gmch_m, \
11538                           current_config->name.gmch_n, \
11539                           current_config->name.link_m, \
11540                           current_config->name.link_n, \
11541                           current_config->alt_name.tu, \
11542                           current_config->alt_name.gmch_m, \
11543                           current_config->alt_name.gmch_n, \
11544                           current_config->alt_name.link_m, \
11545                           current_config->alt_name.link_n, \
11546                           pipe_config->name.tu, \
11547                           pipe_config->name.gmch_m, \
11548                           pipe_config->name.gmch_n, \
11549                           pipe_config->name.link_m, \
11550                           pipe_config->name.link_n); \
11551                 ret = false; \
11552         } \
11553 } while (0)
11554
11555 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
11556         if ((current_config->name ^ pipe_config->name) & (mask)) { \
11557                 pipe_config_err(adjust, __stringify(name), \
11558                           "(%x) (expected %i, found %i)\n", \
11559                           (mask), \
11560                           current_config->name & (mask), \
11561                           pipe_config->name & (mask)); \
11562                 ret = false; \
11563         } \
11564 } while (0)
11565
11566 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
11567         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
11568                 pipe_config_err(adjust, __stringify(name), \
11569                           "(expected %i, found %i)\n", \
11570                           current_config->name, \
11571                           pipe_config->name); \
11572                 ret = false; \
11573         } \
11574 } while (0)
11575
11576 #define PIPE_CONF_QUIRK(quirk)  \
11577         ((current_config->quirks | pipe_config->quirks) & (quirk))
11578
11579         PIPE_CONF_CHECK_I(cpu_transcoder);
11580
11581         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
11582         PIPE_CONF_CHECK_I(fdi_lanes);
11583         PIPE_CONF_CHECK_M_N(fdi_m_n);
11584
11585         PIPE_CONF_CHECK_I(lane_count);
11586         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
11587
11588         if (INTEL_GEN(dev_priv) < 8) {
11589                 PIPE_CONF_CHECK_M_N(dp_m_n);
11590
11591                 if (current_config->has_drrs)
11592                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
11593         } else
11594                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
11595
11596         PIPE_CONF_CHECK_X(output_types);
11597
11598         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
11599         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
11600         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
11601         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
11602         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
11603         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
11604
11605         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
11606         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
11607         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
11608         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
11609         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
11610         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
11611
11612         PIPE_CONF_CHECK_I(pixel_multiplier);
11613         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
11614         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
11615             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11616                 PIPE_CONF_CHECK_BOOL(limited_color_range);
11617
11618         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
11619         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
11620         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
11621         PIPE_CONF_CHECK_BOOL(ycbcr420);
11622
11623         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
11624
11625         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11626                               DRM_MODE_FLAG_INTERLACE);
11627
11628         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
11629                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11630                                       DRM_MODE_FLAG_PHSYNC);
11631                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11632                                       DRM_MODE_FLAG_NHSYNC);
11633                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11634                                       DRM_MODE_FLAG_PVSYNC);
11635                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11636                                       DRM_MODE_FLAG_NVSYNC);
11637         }
11638
11639         PIPE_CONF_CHECK_X(gmch_pfit.control);
11640         /* pfit ratios are autocomputed by the hw on gen4+ */
11641         if (INTEL_GEN(dev_priv) < 4)
11642                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
11643         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
11644
11645         if (!adjust) {
11646                 PIPE_CONF_CHECK_I(pipe_src_w);
11647                 PIPE_CONF_CHECK_I(pipe_src_h);
11648
11649                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
11650                 if (current_config->pch_pfit.enabled) {
11651                         PIPE_CONF_CHECK_X(pch_pfit.pos);
11652                         PIPE_CONF_CHECK_X(pch_pfit.size);
11653                 }
11654
11655                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
11656                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
11657         }
11658
11659         PIPE_CONF_CHECK_BOOL(double_wide);
11660
11661         PIPE_CONF_CHECK_P(shared_dpll);
11662         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
11663         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
11664         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
11665         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
11666         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
11667         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
11668         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
11669         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
11670         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
11671         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
11672         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
11673         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
11674         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
11675         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
11676         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
11677         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
11678         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
11679         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
11680         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
11681         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
11682         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
11683         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
11684         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
11685         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
11686         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
11687         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
11688         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
11689         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
11690         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
11691         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
11692         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
11693
11694         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
11695         PIPE_CONF_CHECK_X(dsi_pll.div);
11696
11697         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
11698                 PIPE_CONF_CHECK_I(pipe_bpp);
11699
11700         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
11701         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
11702
11703         PIPE_CONF_CHECK_I(min_voltage_level);
11704
11705 #undef PIPE_CONF_CHECK_X
11706 #undef PIPE_CONF_CHECK_I
11707 #undef PIPE_CONF_CHECK_BOOL
11708 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
11709 #undef PIPE_CONF_CHECK_P
11710 #undef PIPE_CONF_CHECK_FLAGS
11711 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
11712 #undef PIPE_CONF_QUIRK
11713
11714         return ret;
11715 }
11716
11717 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
11718                                            const struct intel_crtc_state *pipe_config)
11719 {
11720         if (pipe_config->has_pch_encoder) {
11721                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11722                                                             &pipe_config->fdi_m_n);
11723                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
11724
11725                 /*
11726                  * FDI already provided one idea for the dotclock.
11727                  * Yell if the encoder disagrees.
11728                  */
11729                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
11730                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
11731                      fdi_dotclock, dotclock);
11732         }
11733 }
11734
11735 static void verify_wm_state(struct drm_crtc *crtc,
11736                             struct drm_crtc_state *new_state)
11737 {
11738         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11739         struct skl_ddb_allocation hw_ddb, *sw_ddb;
11740         struct skl_pipe_wm hw_wm, *sw_wm;
11741         struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11742         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
11743         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11744         const enum pipe pipe = intel_crtc->pipe;
11745         int plane, level, max_level = ilk_wm_max_level(dev_priv);
11746
11747         if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
11748                 return;
11749
11750         skl_pipe_wm_get_hw_state(crtc, &hw_wm);
11751         sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
11752
11753         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
11754         sw_ddb = &dev_priv->wm.skl_hw.ddb;
11755
11756         if (INTEL_GEN(dev_priv) >= 11)
11757                 if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
11758                         DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
11759                                   sw_ddb->enabled_slices,
11760                                   hw_ddb.enabled_slices);
11761         /* planes */
11762         for_each_universal_plane(dev_priv, pipe, plane) {
11763                 hw_plane_wm = &hw_wm.planes[plane];
11764                 sw_plane_wm = &sw_wm->planes[plane];
11765
11766                 /* Watermarks */
11767                 for (level = 0; level <= max_level; level++) {
11768                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11769                                                 &sw_plane_wm->wm[level]))
11770                                 continue;
11771
11772                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11773                                   pipe_name(pipe), plane + 1, level,
11774                                   sw_plane_wm->wm[level].plane_en,
11775                                   sw_plane_wm->wm[level].plane_res_b,
11776                                   sw_plane_wm->wm[level].plane_res_l,
11777                                   hw_plane_wm->wm[level].plane_en,
11778                                   hw_plane_wm->wm[level].plane_res_b,
11779                                   hw_plane_wm->wm[level].plane_res_l);
11780                 }
11781
11782                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11783                                          &sw_plane_wm->trans_wm)) {
11784                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11785                                   pipe_name(pipe), plane + 1,
11786                                   sw_plane_wm->trans_wm.plane_en,
11787                                   sw_plane_wm->trans_wm.plane_res_b,
11788                                   sw_plane_wm->trans_wm.plane_res_l,
11789                                   hw_plane_wm->trans_wm.plane_en,
11790                                   hw_plane_wm->trans_wm.plane_res_b,
11791                                   hw_plane_wm->trans_wm.plane_res_l);
11792                 }
11793
11794                 /* DDB */
11795                 hw_ddb_entry = &hw_ddb.plane[pipe][plane];
11796                 sw_ddb_entry = &sw_ddb->plane[pipe][plane];
11797
11798                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11799                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
11800                                   pipe_name(pipe), plane + 1,
11801                                   sw_ddb_entry->start, sw_ddb_entry->end,
11802                                   hw_ddb_entry->start, hw_ddb_entry->end);
11803                 }
11804         }
11805
11806         /*
11807          * cursor
11808          * If the cursor plane isn't active, we may not have updated it's ddb
11809          * allocation. In that case since the ddb allocation will be updated
11810          * once the plane becomes visible, we can skip this check
11811          */
11812         if (1) {
11813                 hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
11814                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
11815
11816                 /* Watermarks */
11817                 for (level = 0; level <= max_level; level++) {
11818                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11819                                                 &sw_plane_wm->wm[level]))
11820                                 continue;
11821
11822                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11823                                   pipe_name(pipe), level,
11824                                   sw_plane_wm->wm[level].plane_en,
11825                                   sw_plane_wm->wm[level].plane_res_b,
11826                                   sw_plane_wm->wm[level].plane_res_l,
11827                                   hw_plane_wm->wm[level].plane_en,
11828                                   hw_plane_wm->wm[level].plane_res_b,
11829                                   hw_plane_wm->wm[level].plane_res_l);
11830                 }
11831
11832                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11833                                          &sw_plane_wm->trans_wm)) {
11834                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11835                                   pipe_name(pipe),
11836                                   sw_plane_wm->trans_wm.plane_en,
11837                                   sw_plane_wm->trans_wm.plane_res_b,
11838                                   sw_plane_wm->trans_wm.plane_res_l,
11839                                   hw_plane_wm->trans_wm.plane_en,
11840                                   hw_plane_wm->trans_wm.plane_res_b,
11841                                   hw_plane_wm->trans_wm.plane_res_l);
11842                 }
11843
11844                 /* DDB */
11845                 hw_ddb_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
11846                 sw_ddb_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
11847
11848                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
11849                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
11850                                   pipe_name(pipe),
11851                                   sw_ddb_entry->start, sw_ddb_entry->end,
11852                                   hw_ddb_entry->start, hw_ddb_entry->end);
11853                 }
11854         }
11855 }
11856
11857 static void
11858 verify_connector_state(struct drm_device *dev,
11859                        struct drm_atomic_state *state,
11860                        struct drm_crtc *crtc)
11861 {
11862         struct drm_connector *connector;
11863         struct drm_connector_state *new_conn_state;
11864         int i;
11865
11866         for_each_new_connector_in_state(state, connector, new_conn_state, i) {
11867                 struct drm_encoder *encoder = connector->encoder;
11868                 struct drm_crtc_state *crtc_state = NULL;
11869
11870                 if (new_conn_state->crtc != crtc)
11871                         continue;
11872
11873                 if (crtc)
11874                         crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
11875
11876                 intel_connector_verify_state(crtc_state, new_conn_state);
11877
11878                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
11879                      "connector's atomic encoder doesn't match legacy encoder\n");
11880         }
11881 }
11882
11883 static void
11884 verify_encoder_state(struct drm_device *dev, struct drm_atomic_state *state)
11885 {
11886         struct intel_encoder *encoder;
11887         struct drm_connector *connector;
11888         struct drm_connector_state *old_conn_state, *new_conn_state;
11889         int i;
11890
11891         for_each_intel_encoder(dev, encoder) {
11892                 bool enabled = false, found = false;
11893                 enum pipe pipe;
11894
11895                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
11896                               encoder->base.base.id,
11897                               encoder->base.name);
11898
11899                 for_each_oldnew_connector_in_state(state, connector, old_conn_state,
11900                                                    new_conn_state, i) {
11901                         if (old_conn_state->best_encoder == &encoder->base)
11902                                 found = true;
11903
11904                         if (new_conn_state->best_encoder != &encoder->base)
11905                                 continue;
11906                         found = enabled = true;
11907
11908                         I915_STATE_WARN(new_conn_state->crtc !=
11909                                         encoder->base.crtc,
11910                              "connector's crtc doesn't match encoder crtc\n");
11911                 }
11912
11913                 if (!found)
11914                         continue;
11915
11916                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
11917                      "encoder's enabled state mismatch "
11918                      "(expected %i, found %i)\n",
11919                      !!encoder->base.crtc, enabled);
11920
11921                 if (!encoder->base.crtc) {
11922                         bool active;
11923
11924                         active = encoder->get_hw_state(encoder, &pipe);
11925                         I915_STATE_WARN(active,
11926                              "encoder detached but still enabled on pipe %c.\n",
11927                              pipe_name(pipe));
11928                 }
11929         }
11930 }
11931
11932 static void
11933 verify_crtc_state(struct drm_crtc *crtc,
11934                   struct drm_crtc_state *old_crtc_state,
11935                   struct drm_crtc_state *new_crtc_state)
11936 {
11937         struct drm_device *dev = crtc->dev;
11938         struct drm_i915_private *dev_priv = to_i915(dev);
11939         struct intel_encoder *encoder;
11940         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11941         struct intel_crtc_state *pipe_config, *sw_config;
11942         struct drm_atomic_state *old_state;
11943         bool active;
11944
11945         old_state = old_crtc_state->state;
11946         __drm_atomic_helper_crtc_destroy_state(old_crtc_state);
11947         pipe_config = to_intel_crtc_state(old_crtc_state);
11948         memset(pipe_config, 0, sizeof(*pipe_config));
11949         pipe_config->base.crtc = crtc;
11950         pipe_config->base.state = old_state;
11951
11952         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
11953
11954         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
11955
11956         /* we keep both pipes enabled on 830 */
11957         if (IS_I830(dev_priv))
11958                 active = new_crtc_state->active;
11959
11960         I915_STATE_WARN(new_crtc_state->active != active,
11961              "crtc active state doesn't match with hw state "
11962              "(expected %i, found %i)\n", new_crtc_state->active, active);
11963
11964         I915_STATE_WARN(intel_crtc->active != new_crtc_state->active,
11965              "transitional active state does not match atomic hw state "
11966              "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active);
11967
11968         for_each_encoder_on_crtc(dev, crtc, encoder) {
11969                 enum pipe pipe;
11970
11971                 active = encoder->get_hw_state(encoder, &pipe);
11972                 I915_STATE_WARN(active != new_crtc_state->active,
11973                         "[ENCODER:%i] active %i with crtc active %i\n",
11974                         encoder->base.base.id, active, new_crtc_state->active);
11975
11976                 I915_STATE_WARN(active && intel_crtc->pipe != pipe,
11977                                 "Encoder connected to wrong pipe %c\n",
11978                                 pipe_name(pipe));
11979
11980                 if (active)
11981                         encoder->get_config(encoder, pipe_config);
11982         }
11983
11984         intel_crtc_compute_pixel_rate(pipe_config);
11985
11986         if (!new_crtc_state->active)
11987                 return;
11988
11989         intel_pipe_config_sanity_check(dev_priv, pipe_config);
11990
11991         sw_config = to_intel_crtc_state(new_crtc_state);
11992         if (!intel_pipe_config_compare(dev_priv, sw_config,
11993                                        pipe_config, false)) {
11994                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
11995                 intel_dump_pipe_config(intel_crtc, pipe_config,
11996                                        "[hw state]");
11997                 intel_dump_pipe_config(intel_crtc, sw_config,
11998                                        "[sw state]");
11999         }
12000 }
12001
12002 static void
12003 intel_verify_planes(struct intel_atomic_state *state)
12004 {
12005         struct intel_plane *plane;
12006         const struct intel_plane_state *plane_state;
12007         int i;
12008
12009         for_each_new_intel_plane_in_state(state, plane,
12010                                           plane_state, i)
12011                 assert_plane(plane, plane_state->base.visible);
12012 }
12013
12014 static void
12015 verify_single_dpll_state(struct drm_i915_private *dev_priv,
12016                          struct intel_shared_dpll *pll,
12017                          struct drm_crtc *crtc,
12018                          struct drm_crtc_state *new_state)
12019 {
12020         struct intel_dpll_hw_state dpll_hw_state;
12021         unsigned int crtc_mask;
12022         bool active;
12023
12024         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12025
12026         DRM_DEBUG_KMS("%s\n", pll->info->name);
12027
12028         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
12029
12030         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
12031                 I915_STATE_WARN(!pll->on && pll->active_mask,
12032                      "pll in active use but not on in sw tracking\n");
12033                 I915_STATE_WARN(pll->on && !pll->active_mask,
12034                      "pll is on but not used by any active crtc\n");
12035                 I915_STATE_WARN(pll->on != active,
12036                      "pll on state mismatch (expected %i, found %i)\n",
12037                      pll->on, active);
12038         }
12039
12040         if (!crtc) {
12041                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
12042                                 "more active pll users than references: %x vs %x\n",
12043                                 pll->active_mask, pll->state.crtc_mask);
12044
12045                 return;
12046         }
12047
12048         crtc_mask = drm_crtc_mask(crtc);
12049
12050         if (new_state->active)
12051                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
12052                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
12053                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12054         else
12055                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12056                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
12057                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12058
12059         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
12060                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
12061                         crtc_mask, pll->state.crtc_mask);
12062
12063         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
12064                                           &dpll_hw_state,
12065                                           sizeof(dpll_hw_state)),
12066                         "pll hw state mismatch\n");
12067 }
12068
12069 static void
12070 verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
12071                          struct drm_crtc_state *old_crtc_state,
12072                          struct drm_crtc_state *new_crtc_state)
12073 {
12074         struct drm_i915_private *dev_priv = to_i915(dev);
12075         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
12076         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
12077
12078         if (new_state->shared_dpll)
12079                 verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state);
12080
12081         if (old_state->shared_dpll &&
12082             old_state->shared_dpll != new_state->shared_dpll) {
12083                 unsigned int crtc_mask = drm_crtc_mask(crtc);
12084                 struct intel_shared_dpll *pll = old_state->shared_dpll;
12085
12086                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12087                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
12088                                 pipe_name(drm_crtc_index(crtc)));
12089                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
12090                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
12091                                 pipe_name(drm_crtc_index(crtc)));
12092         }
12093 }
12094
12095 static void
12096 intel_modeset_verify_crtc(struct drm_crtc *crtc,
12097                           struct drm_atomic_state *state,
12098                           struct drm_crtc_state *old_state,
12099                           struct drm_crtc_state *new_state)
12100 {
12101         if (!needs_modeset(new_state) &&
12102             !to_intel_crtc_state(new_state)->update_pipe)
12103                 return;
12104
12105         verify_wm_state(crtc, new_state);
12106         verify_connector_state(crtc->dev, state, crtc);
12107         verify_crtc_state(crtc, old_state, new_state);
12108         verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
12109 }
12110
12111 static void
12112 verify_disabled_dpll_state(struct drm_device *dev)
12113 {
12114         struct drm_i915_private *dev_priv = to_i915(dev);
12115         int i;
12116
12117         for (i = 0; i < dev_priv->num_shared_dpll; i++)
12118                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
12119 }
12120
12121 static void
12122 intel_modeset_verify_disabled(struct drm_device *dev,
12123                               struct drm_atomic_state *state)
12124 {
12125         verify_encoder_state(dev, state);
12126         verify_connector_state(dev, state, NULL);
12127         verify_disabled_dpll_state(dev);
12128 }
12129
12130 static void update_scanline_offset(struct intel_crtc *crtc)
12131 {
12132         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12133
12134         /*
12135          * The scanline counter increments at the leading edge of hsync.
12136          *
12137          * On most platforms it starts counting from vtotal-1 on the
12138          * first active line. That means the scanline counter value is
12139          * always one less than what we would expect. Ie. just after
12140          * start of vblank, which also occurs at start of hsync (on the
12141          * last active line), the scanline counter will read vblank_start-1.
12142          *
12143          * On gen2 the scanline counter starts counting from 1 instead
12144          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12145          * to keep the value positive), instead of adding one.
12146          *
12147          * On HSW+ the behaviour of the scanline counter depends on the output
12148          * type. For DP ports it behaves like most other platforms, but on HDMI
12149          * there's an extra 1 line difference. So we need to add two instead of
12150          * one to the value.
12151          *
12152          * On VLV/CHV DSI the scanline counter would appear to increment
12153          * approx. 1/3 of a scanline before start of vblank. Unfortunately
12154          * that means we can't tell whether we're in vblank or not while
12155          * we're on that particular line. We must still set scanline_offset
12156          * to 1 so that the vblank timestamps come out correct when we query
12157          * the scanline counter from within the vblank interrupt handler.
12158          * However if queried just before the start of vblank we'll get an
12159          * answer that's slightly in the future.
12160          */
12161         if (IS_GEN2(dev_priv)) {
12162                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
12163                 int vtotal;
12164
12165                 vtotal = adjusted_mode->crtc_vtotal;
12166                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12167                         vtotal /= 2;
12168
12169                 crtc->scanline_offset = vtotal - 1;
12170         } else if (HAS_DDI(dev_priv) &&
12171                    intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) {
12172                 crtc->scanline_offset = 2;
12173         } else
12174                 crtc->scanline_offset = 1;
12175 }
12176
12177 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12178 {
12179         struct drm_device *dev = state->dev;
12180         struct drm_i915_private *dev_priv = to_i915(dev);
12181         struct drm_crtc *crtc;
12182         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12183         int i;
12184
12185         if (!dev_priv->display.crtc_compute_clock)
12186                 return;
12187
12188         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12189                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12190                 struct intel_shared_dpll *old_dpll =
12191                         to_intel_crtc_state(old_crtc_state)->shared_dpll;
12192
12193                 if (!needs_modeset(new_crtc_state))
12194                         continue;
12195
12196                 to_intel_crtc_state(new_crtc_state)->shared_dpll = NULL;
12197
12198                 if (!old_dpll)
12199                         continue;
12200
12201                 intel_release_shared_dpll(old_dpll, intel_crtc, state);
12202         }
12203 }
12204
12205 /*
12206  * This implements the workaround described in the "notes" section of the mode
12207  * set sequence documentation. When going from no pipes or single pipe to
12208  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12209  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12210  */
12211 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12212 {
12213         struct drm_crtc_state *crtc_state;
12214         struct intel_crtc *intel_crtc;
12215         struct drm_crtc *crtc;
12216         struct intel_crtc_state *first_crtc_state = NULL;
12217         struct intel_crtc_state *other_crtc_state = NULL;
12218         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12219         int i;
12220
12221         /* look at all crtc's that are going to be enabled in during modeset */
12222         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
12223                 intel_crtc = to_intel_crtc(crtc);
12224
12225                 if (!crtc_state->active || !needs_modeset(crtc_state))
12226                         continue;
12227
12228                 if (first_crtc_state) {
12229                         other_crtc_state = to_intel_crtc_state(crtc_state);
12230                         break;
12231                 } else {
12232                         first_crtc_state = to_intel_crtc_state(crtc_state);
12233                         first_pipe = intel_crtc->pipe;
12234                 }
12235         }
12236
12237         /* No workaround needed? */
12238         if (!first_crtc_state)
12239                 return 0;
12240
12241         /* w/a possibly needed, check how many crtc's are already enabled. */
12242         for_each_intel_crtc(state->dev, intel_crtc) {
12243                 struct intel_crtc_state *pipe_config;
12244
12245                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
12246                 if (IS_ERR(pipe_config))
12247                         return PTR_ERR(pipe_config);
12248
12249                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
12250
12251                 if (!pipe_config->base.active ||
12252                     needs_modeset(&pipe_config->base))
12253                         continue;
12254
12255                 /* 2 or more enabled crtcs means no need for w/a */
12256                 if (enabled_pipe != INVALID_PIPE)
12257                         return 0;
12258
12259                 enabled_pipe = intel_crtc->pipe;
12260         }
12261
12262         if (enabled_pipe != INVALID_PIPE)
12263                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
12264         else if (other_crtc_state)
12265                 other_crtc_state->hsw_workaround_pipe = first_pipe;
12266
12267         return 0;
12268 }
12269
12270 static int intel_lock_all_pipes(struct drm_atomic_state *state)
12271 {
12272         struct drm_crtc *crtc;
12273
12274         /* Add all pipes to the state */
12275         for_each_crtc(state->dev, crtc) {
12276                 struct drm_crtc_state *crtc_state;
12277
12278                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12279                 if (IS_ERR(crtc_state))
12280                         return PTR_ERR(crtc_state);
12281         }
12282
12283         return 0;
12284 }
12285
12286 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
12287 {
12288         struct drm_crtc *crtc;
12289
12290         /*
12291          * Add all pipes to the state, and force
12292          * a modeset on all the active ones.
12293          */
12294         for_each_crtc(state->dev, crtc) {
12295                 struct drm_crtc_state *crtc_state;
12296                 int ret;
12297
12298                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12299                 if (IS_ERR(crtc_state))
12300                         return PTR_ERR(crtc_state);
12301
12302                 if (!crtc_state->active || needs_modeset(crtc_state))
12303                         continue;
12304
12305                 crtc_state->mode_changed = true;
12306
12307                 ret = drm_atomic_add_affected_connectors(state, crtc);
12308                 if (ret)
12309                         return ret;
12310
12311                 ret = drm_atomic_add_affected_planes(state, crtc);
12312                 if (ret)
12313                         return ret;
12314         }
12315
12316         return 0;
12317 }
12318
12319 static int intel_modeset_checks(struct drm_atomic_state *state)
12320 {
12321         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12322         struct drm_i915_private *dev_priv = to_i915(state->dev);
12323         struct drm_crtc *crtc;
12324         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12325         int ret = 0, i;
12326
12327         if (!check_digital_port_conflicts(state)) {
12328                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
12329                 return -EINVAL;
12330         }
12331
12332         intel_state->modeset = true;
12333         intel_state->active_crtcs = dev_priv->active_crtcs;
12334         intel_state->cdclk.logical = dev_priv->cdclk.logical;
12335         intel_state->cdclk.actual = dev_priv->cdclk.actual;
12336
12337         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12338                 if (new_crtc_state->active)
12339                         intel_state->active_crtcs |= 1 << i;
12340                 else
12341                         intel_state->active_crtcs &= ~(1 << i);
12342
12343                 if (old_crtc_state->active != new_crtc_state->active)
12344                         intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
12345         }
12346
12347         /*
12348          * See if the config requires any additional preparation, e.g.
12349          * to adjust global state with pipes off.  We need to do this
12350          * here so we can get the modeset_pipe updated config for the new
12351          * mode set on this crtc.  For other crtcs we need to use the
12352          * adjusted_mode bits in the crtc directly.
12353          */
12354         if (dev_priv->display.modeset_calc_cdclk) {
12355                 ret = dev_priv->display.modeset_calc_cdclk(state);
12356                 if (ret < 0)
12357                         return ret;
12358
12359                 /*
12360                  * Writes to dev_priv->cdclk.logical must protected by
12361                  * holding all the crtc locks, even if we don't end up
12362                  * touching the hardware
12363                  */
12364                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
12365                                         &intel_state->cdclk.logical)) {
12366                         ret = intel_lock_all_pipes(state);
12367                         if (ret < 0)
12368                                 return ret;
12369                 }
12370
12371                 /* All pipes must be switched off while we change the cdclk. */
12372                 if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
12373                                               &intel_state->cdclk.actual)) {
12374                         ret = intel_modeset_all_pipes(state);
12375                         if (ret < 0)
12376                                 return ret;
12377                 }
12378
12379                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
12380                               intel_state->cdclk.logical.cdclk,
12381                               intel_state->cdclk.actual.cdclk);
12382                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
12383                               intel_state->cdclk.logical.voltage_level,
12384                               intel_state->cdclk.actual.voltage_level);
12385         } else {
12386                 to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
12387         }
12388
12389         intel_modeset_clear_plls(state);
12390
12391         if (IS_HASWELL(dev_priv))
12392                 return haswell_mode_set_planes_workaround(state);
12393
12394         return 0;
12395 }
12396
12397 /*
12398  * Handle calculation of various watermark data at the end of the atomic check
12399  * phase.  The code here should be run after the per-crtc and per-plane 'check'
12400  * handlers to ensure that all derived state has been updated.
12401  */
12402 static int calc_watermark_data(struct drm_atomic_state *state)
12403 {
12404         struct drm_device *dev = state->dev;
12405         struct drm_i915_private *dev_priv = to_i915(dev);
12406
12407         /* Is there platform-specific watermark information to calculate? */
12408         if (dev_priv->display.compute_global_watermarks)
12409                 return dev_priv->display.compute_global_watermarks(state);
12410
12411         return 0;
12412 }
12413
12414 /**
12415  * intel_atomic_check - validate state object
12416  * @dev: drm device
12417  * @state: state to validate
12418  */
12419 static int intel_atomic_check(struct drm_device *dev,
12420                               struct drm_atomic_state *state)
12421 {
12422         struct drm_i915_private *dev_priv = to_i915(dev);
12423         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12424         struct drm_crtc *crtc;
12425         struct drm_crtc_state *old_crtc_state, *crtc_state;
12426         int ret, i;
12427         bool any_ms = false;
12428
12429         /* Catch I915_MODE_FLAG_INHERITED */
12430         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
12431                                       crtc_state, i) {
12432                 if (crtc_state->mode.private_flags !=
12433                     old_crtc_state->mode.private_flags)
12434                         crtc_state->mode_changed = true;
12435         }
12436
12437         ret = drm_atomic_helper_check_modeset(dev, state);
12438         if (ret)
12439                 return ret;
12440
12441         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) {
12442                 struct intel_crtc_state *pipe_config =
12443                         to_intel_crtc_state(crtc_state);
12444
12445                 if (!needs_modeset(crtc_state))
12446                         continue;
12447
12448                 if (!crtc_state->enable) {
12449                         any_ms = true;
12450                         continue;
12451                 }
12452
12453                 ret = intel_modeset_pipe_config(crtc, pipe_config);
12454                 if (ret) {
12455                         intel_dump_pipe_config(to_intel_crtc(crtc),
12456                                                pipe_config, "[failed]");
12457                         return ret;
12458                 }
12459
12460                 if (i915_modparams.fastboot &&
12461                     intel_pipe_config_compare(dev_priv,
12462                                         to_intel_crtc_state(old_crtc_state),
12463                                         pipe_config, true)) {
12464                         crtc_state->mode_changed = false;
12465                         pipe_config->update_pipe = true;
12466                 }
12467
12468                 if (needs_modeset(crtc_state))
12469                         any_ms = true;
12470
12471                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
12472                                        needs_modeset(crtc_state) ?
12473                                        "[modeset]" : "[fastset]");
12474         }
12475
12476         if (any_ms) {
12477                 ret = intel_modeset_checks(state);
12478
12479                 if (ret)
12480                         return ret;
12481         } else {
12482                 intel_state->cdclk.logical = dev_priv->cdclk.logical;
12483         }
12484
12485         ret = drm_atomic_helper_check_planes(dev, state);
12486         if (ret)
12487                 return ret;
12488
12489         intel_fbc_choose_crtc(dev_priv, intel_state);
12490         return calc_watermark_data(state);
12491 }
12492
12493 static int intel_atomic_prepare_commit(struct drm_device *dev,
12494                                        struct drm_atomic_state *state)
12495 {
12496         return drm_atomic_helper_prepare_planes(dev, state);
12497 }
12498
12499 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
12500 {
12501         struct drm_device *dev = crtc->base.dev;
12502
12503         if (!dev->max_vblank_count)
12504                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
12505
12506         return dev->driver->get_vblank_counter(dev, crtc->pipe);
12507 }
12508
12509 static void intel_update_crtc(struct drm_crtc *crtc,
12510                               struct drm_atomic_state *state,
12511                               struct drm_crtc_state *old_crtc_state,
12512                               struct drm_crtc_state *new_crtc_state)
12513 {
12514         struct drm_device *dev = crtc->dev;
12515         struct drm_i915_private *dev_priv = to_i915(dev);
12516         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12517         struct intel_crtc_state *old_intel_cstate = to_intel_crtc_state(old_crtc_state);
12518         struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state);
12519         bool modeset = needs_modeset(new_crtc_state);
12520         struct intel_plane_state *new_plane_state =
12521                 intel_atomic_get_new_plane_state(to_intel_atomic_state(state),
12522                                                  to_intel_plane(crtc->primary));
12523
12524         if (modeset) {
12525                 update_scanline_offset(intel_crtc);
12526                 dev_priv->display.crtc_enable(pipe_config, state);
12527
12528                 /* vblanks work again, re-enable pipe CRC. */
12529                 intel_crtc_enable_pipe_crc(intel_crtc);
12530         } else {
12531                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
12532                                        pipe_config);
12533         }
12534
12535         if (new_plane_state)
12536                 intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
12537
12538         intel_begin_crtc_commit(crtc, old_crtc_state);
12539
12540         intel_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc,
12541                                     old_intel_cstate, pipe_config);
12542
12543         intel_finish_crtc_commit(crtc, old_crtc_state);
12544 }
12545
12546 static void intel_update_crtcs(struct drm_atomic_state *state)
12547 {
12548         struct drm_crtc *crtc;
12549         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12550         int i;
12551
12552         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12553                 if (!new_crtc_state->active)
12554                         continue;
12555
12556                 intel_update_crtc(crtc, state, old_crtc_state,
12557                                   new_crtc_state);
12558         }
12559 }
12560
12561 static void skl_update_crtcs(struct drm_atomic_state *state)
12562 {
12563         struct drm_i915_private *dev_priv = to_i915(state->dev);
12564         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12565         struct drm_crtc *crtc;
12566         struct intel_crtc *intel_crtc;
12567         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12568         struct intel_crtc_state *cstate;
12569         unsigned int updated = 0;
12570         bool progress;
12571         enum pipe pipe;
12572         int i;
12573         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
12574         u8 required_slices = intel_state->wm_results.ddb.enabled_slices;
12575
12576         const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {};
12577
12578         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
12579                 /* ignore allocations for crtc's that have been turned off. */
12580                 if (new_crtc_state->active)
12581                         entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
12582
12583         /* If 2nd DBuf slice required, enable it here */
12584         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
12585                 icl_dbuf_slices_update(dev_priv, required_slices);
12586
12587         /*
12588          * Whenever the number of active pipes changes, we need to make sure we
12589          * update the pipes in the right order so that their ddb allocations
12590          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
12591          * cause pipe underruns and other bad stuff.
12592          */
12593         do {
12594                 progress = false;
12595
12596                 for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12597                         bool vbl_wait = false;
12598                         unsigned int cmask = drm_crtc_mask(crtc);
12599
12600                         intel_crtc = to_intel_crtc(crtc);
12601                         cstate = to_intel_crtc_state(new_crtc_state);
12602                         pipe = intel_crtc->pipe;
12603
12604                         if (updated & cmask || !cstate->base.active)
12605                                 continue;
12606
12607                         if (skl_ddb_allocation_overlaps(dev_priv,
12608                                                         entries,
12609                                                         &cstate->wm.skl.ddb,
12610                                                         i))
12611                                 continue;
12612
12613                         updated |= cmask;
12614                         entries[i] = &cstate->wm.skl.ddb;
12615
12616                         /*
12617                          * If this is an already active pipe, it's DDB changed,
12618                          * and this isn't the last pipe that needs updating
12619                          * then we need to wait for a vblank to pass for the
12620                          * new ddb allocation to take effect.
12621                          */
12622                         if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
12623                                                  &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
12624                             !new_crtc_state->active_changed &&
12625                             intel_state->wm_results.dirty_pipes != updated)
12626                                 vbl_wait = true;
12627
12628                         intel_update_crtc(crtc, state, old_crtc_state,
12629                                           new_crtc_state);
12630
12631                         if (vbl_wait)
12632                                 intel_wait_for_vblank(dev_priv, pipe);
12633
12634                         progress = true;
12635                 }
12636         } while (progress);
12637
12638         /* If 2nd DBuf slice is no more required disable it */
12639         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
12640                 icl_dbuf_slices_update(dev_priv, required_slices);
12641 }
12642
12643 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
12644 {
12645         struct intel_atomic_state *state, *next;
12646         struct llist_node *freed;
12647
12648         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
12649         llist_for_each_entry_safe(state, next, freed, freed)
12650                 drm_atomic_state_put(&state->base);
12651 }
12652
12653 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
12654 {
12655         struct drm_i915_private *dev_priv =
12656                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
12657
12658         intel_atomic_helper_free_state(dev_priv);
12659 }
12660
12661 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
12662 {
12663         struct wait_queue_entry wait_fence, wait_reset;
12664         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
12665
12666         init_wait_entry(&wait_fence, 0);
12667         init_wait_entry(&wait_reset, 0);
12668         for (;;) {
12669                 prepare_to_wait(&intel_state->commit_ready.wait,
12670                                 &wait_fence, TASK_UNINTERRUPTIBLE);
12671                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
12672                                 &wait_reset, TASK_UNINTERRUPTIBLE);
12673
12674
12675                 if (i915_sw_fence_done(&intel_state->commit_ready)
12676                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
12677                         break;
12678
12679                 schedule();
12680         }
12681         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
12682         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
12683 }
12684
12685 static void intel_atomic_cleanup_work(struct work_struct *work)
12686 {
12687         struct drm_atomic_state *state =
12688                 container_of(work, struct drm_atomic_state, commit_work);
12689         struct drm_i915_private *i915 = to_i915(state->dev);
12690
12691         drm_atomic_helper_cleanup_planes(&i915->drm, state);
12692         drm_atomic_helper_commit_cleanup_done(state);
12693         drm_atomic_state_put(state);
12694
12695         intel_atomic_helper_free_state(i915);
12696 }
12697
12698 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
12699 {
12700         struct drm_device *dev = state->dev;
12701         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12702         struct drm_i915_private *dev_priv = to_i915(dev);
12703         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12704         struct intel_crtc_state *new_intel_crtc_state, *old_intel_crtc_state;
12705         struct drm_crtc *crtc;
12706         struct intel_crtc *intel_crtc;
12707         u64 put_domains[I915_MAX_PIPES] = {};
12708         int i;
12709
12710         intel_atomic_commit_fence_wait(intel_state);
12711
12712         drm_atomic_helper_wait_for_dependencies(state);
12713
12714         if (intel_state->modeset)
12715                 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
12716
12717         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12718                 old_intel_crtc_state = to_intel_crtc_state(old_crtc_state);
12719                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
12720                 intel_crtc = to_intel_crtc(crtc);
12721
12722                 if (needs_modeset(new_crtc_state) ||
12723                     to_intel_crtc_state(new_crtc_state)->update_pipe) {
12724
12725                         put_domains[intel_crtc->pipe] =
12726                                 modeset_get_crtc_power_domains(crtc,
12727                                         new_intel_crtc_state);
12728                 }
12729
12730                 if (!needs_modeset(new_crtc_state))
12731                         continue;
12732
12733                 intel_pre_plane_update(old_intel_crtc_state, new_intel_crtc_state);
12734
12735                 if (old_crtc_state->active) {
12736                         intel_crtc_disable_planes(intel_crtc, old_intel_crtc_state->active_planes);
12737
12738                         /*
12739                          * We need to disable pipe CRC before disabling the pipe,
12740                          * or we race against vblank off.
12741                          */
12742                         intel_crtc_disable_pipe_crc(intel_crtc);
12743
12744                         dev_priv->display.crtc_disable(old_intel_crtc_state, state);
12745                         intel_crtc->active = false;
12746                         intel_fbc_disable(intel_crtc);
12747                         intel_disable_shared_dpll(intel_crtc);
12748
12749                         /*
12750                          * Underruns don't always raise
12751                          * interrupts, so check manually.
12752                          */
12753                         intel_check_cpu_fifo_underruns(dev_priv);
12754                         intel_check_pch_fifo_underruns(dev_priv);
12755
12756                         if (!new_crtc_state->active) {
12757                                 /*
12758                                  * Make sure we don't call initial_watermarks
12759                                  * for ILK-style watermark updates.
12760                                  *
12761                                  * No clue what this is supposed to achieve.
12762                                  */
12763                                 if (INTEL_GEN(dev_priv) >= 9)
12764                                         dev_priv->display.initial_watermarks(intel_state,
12765                                                                              new_intel_crtc_state);
12766                         }
12767                 }
12768         }
12769
12770         /* FIXME: Eventually get rid of our intel_crtc->config pointer */
12771         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
12772                 to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
12773
12774         if (intel_state->modeset) {
12775                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
12776
12777                 intel_set_cdclk(dev_priv, &dev_priv->cdclk.actual);
12778
12779                 /*
12780                  * SKL workaround: bspec recommends we disable the SAGV when we
12781                  * have more then one pipe enabled
12782                  */
12783                 if (!intel_can_enable_sagv(state))
12784                         intel_disable_sagv(dev_priv);
12785
12786                 intel_modeset_verify_disabled(dev, state);
12787         }
12788
12789         /* Complete the events for pipes that have now been disabled */
12790         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
12791                 bool modeset = needs_modeset(new_crtc_state);
12792
12793                 /* Complete events for now disable pipes here. */
12794                 if (modeset && !new_crtc_state->active && new_crtc_state->event) {
12795                         spin_lock_irq(&dev->event_lock);
12796                         drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
12797                         spin_unlock_irq(&dev->event_lock);
12798
12799                         new_crtc_state->event = NULL;
12800                 }
12801         }
12802
12803         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
12804         dev_priv->display.update_crtcs(state);
12805
12806         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
12807          * already, but still need the state for the delayed optimization. To
12808          * fix this:
12809          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
12810          * - schedule that vblank worker _before_ calling hw_done
12811          * - at the start of commit_tail, cancel it _synchrously
12812          * - switch over to the vblank wait helper in the core after that since
12813          *   we don't need out special handling any more.
12814          */
12815         drm_atomic_helper_wait_for_flip_done(dev, state);
12816
12817         /*
12818          * Now that the vblank has passed, we can go ahead and program the
12819          * optimal watermarks on platforms that need two-step watermark
12820          * programming.
12821          *
12822          * TODO: Move this (and other cleanup) to an async worker eventually.
12823          */
12824         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
12825                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
12826
12827                 if (dev_priv->display.optimize_watermarks)
12828                         dev_priv->display.optimize_watermarks(intel_state,
12829                                                               new_intel_crtc_state);
12830         }
12831
12832         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12833                 intel_post_plane_update(to_intel_crtc_state(old_crtc_state));
12834
12835                 if (put_domains[i])
12836                         modeset_put_power_domains(dev_priv, put_domains[i]);
12837
12838                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
12839         }
12840
12841         if (intel_state->modeset)
12842                 intel_verify_planes(intel_state);
12843
12844         if (intel_state->modeset && intel_can_enable_sagv(state))
12845                 intel_enable_sagv(dev_priv);
12846
12847         drm_atomic_helper_commit_hw_done(state);
12848
12849         if (intel_state->modeset) {
12850                 /* As one of the primary mmio accessors, KMS has a high
12851                  * likelihood of triggering bugs in unclaimed access. After we
12852                  * finish modesetting, see if an error has been flagged, and if
12853                  * so enable debugging for the next modeset - and hope we catch
12854                  * the culprit.
12855                  */
12856                 intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
12857                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
12858         }
12859
12860         /*
12861          * Defer the cleanup of the old state to a separate worker to not
12862          * impede the current task (userspace for blocking modesets) that
12863          * are executed inline. For out-of-line asynchronous modesets/flips,
12864          * deferring to a new worker seems overkill, but we would place a
12865          * schedule point (cond_resched()) here anyway to keep latencies
12866          * down.
12867          */
12868         INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
12869         queue_work(system_highpri_wq, &state->commit_work);
12870 }
12871
12872 static void intel_atomic_commit_work(struct work_struct *work)
12873 {
12874         struct drm_atomic_state *state =
12875                 container_of(work, struct drm_atomic_state, commit_work);
12876
12877         intel_atomic_commit_tail(state);
12878 }
12879
12880 static int __i915_sw_fence_call
12881 intel_atomic_commit_ready(struct i915_sw_fence *fence,
12882                           enum i915_sw_fence_notify notify)
12883 {
12884         struct intel_atomic_state *state =
12885                 container_of(fence, struct intel_atomic_state, commit_ready);
12886
12887         switch (notify) {
12888         case FENCE_COMPLETE:
12889                 /* we do blocking waits in the worker, nothing to do here */
12890                 break;
12891         case FENCE_FREE:
12892                 {
12893                         struct intel_atomic_helper *helper =
12894                                 &to_i915(state->base.dev)->atomic_helper;
12895
12896                         if (llist_add(&state->freed, &helper->free_list))
12897                                 schedule_work(&helper->free_work);
12898                         break;
12899                 }
12900         }
12901
12902         return NOTIFY_DONE;
12903 }
12904
12905 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
12906 {
12907         struct drm_plane_state *old_plane_state, *new_plane_state;
12908         struct drm_plane *plane;
12909         int i;
12910
12911         for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
12912                 i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
12913                                   intel_fb_obj(new_plane_state->fb),
12914                                   to_intel_plane(plane)->frontbuffer_bit);
12915 }
12916
12917 /**
12918  * intel_atomic_commit - commit validated state object
12919  * @dev: DRM device
12920  * @state: the top-level driver state object
12921  * @nonblock: nonblocking commit
12922  *
12923  * This function commits a top-level state object that has been validated
12924  * with drm_atomic_helper_check().
12925  *
12926  * RETURNS
12927  * Zero for success or -errno.
12928  */
12929 static int intel_atomic_commit(struct drm_device *dev,
12930                                struct drm_atomic_state *state,
12931                                bool nonblock)
12932 {
12933         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12934         struct drm_i915_private *dev_priv = to_i915(dev);
12935         int ret = 0;
12936
12937         drm_atomic_state_get(state);
12938         i915_sw_fence_init(&intel_state->commit_ready,
12939                            intel_atomic_commit_ready);
12940
12941         /*
12942          * The intel_legacy_cursor_update() fast path takes care
12943          * of avoiding the vblank waits for simple cursor
12944          * movement and flips. For cursor on/off and size changes,
12945          * we want to perform the vblank waits so that watermark
12946          * updates happen during the correct frames. Gen9+ have
12947          * double buffered watermarks and so shouldn't need this.
12948          *
12949          * Unset state->legacy_cursor_update before the call to
12950          * drm_atomic_helper_setup_commit() because otherwise
12951          * drm_atomic_helper_wait_for_flip_done() is a noop and
12952          * we get FIFO underruns because we didn't wait
12953          * for vblank.
12954          *
12955          * FIXME doing watermarks and fb cleanup from a vblank worker
12956          * (assuming we had any) would solve these problems.
12957          */
12958         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
12959                 struct intel_crtc_state *new_crtc_state;
12960                 struct intel_crtc *crtc;
12961                 int i;
12962
12963                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
12964                         if (new_crtc_state->wm.need_postvbl_update ||
12965                             new_crtc_state->update_wm_post)
12966                                 state->legacy_cursor_update = false;
12967         }
12968
12969         ret = intel_atomic_prepare_commit(dev, state);
12970         if (ret) {
12971                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
12972                 i915_sw_fence_commit(&intel_state->commit_ready);
12973                 return ret;
12974         }
12975
12976         ret = drm_atomic_helper_setup_commit(state, nonblock);
12977         if (!ret)
12978                 ret = drm_atomic_helper_swap_state(state, true);
12979
12980         if (ret) {
12981                 i915_sw_fence_commit(&intel_state->commit_ready);
12982
12983                 drm_atomic_helper_cleanup_planes(dev, state);
12984                 return ret;
12985         }
12986         dev_priv->wm.distrust_bios_wm = false;
12987         intel_shared_dpll_swap_state(state);
12988         intel_atomic_track_fbs(state);
12989
12990         if (intel_state->modeset) {
12991                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
12992                        sizeof(intel_state->min_cdclk));
12993                 memcpy(dev_priv->min_voltage_level,
12994                        intel_state->min_voltage_level,
12995                        sizeof(intel_state->min_voltage_level));
12996                 dev_priv->active_crtcs = intel_state->active_crtcs;
12997                 dev_priv->cdclk.logical = intel_state->cdclk.logical;
12998                 dev_priv->cdclk.actual = intel_state->cdclk.actual;
12999         }
13000
13001         drm_atomic_state_get(state);
13002         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
13003
13004         i915_sw_fence_commit(&intel_state->commit_ready);
13005         if (nonblock && intel_state->modeset) {
13006                 queue_work(dev_priv->modeset_wq, &state->commit_work);
13007         } else if (nonblock) {
13008                 queue_work(system_unbound_wq, &state->commit_work);
13009         } else {
13010                 if (intel_state->modeset)
13011                         flush_workqueue(dev_priv->modeset_wq);
13012                 intel_atomic_commit_tail(state);
13013         }
13014
13015         return 0;
13016 }
13017
13018 static const struct drm_crtc_funcs intel_crtc_funcs = {
13019         .gamma_set = drm_atomic_helper_legacy_gamma_set,
13020         .set_config = drm_atomic_helper_set_config,
13021         .destroy = intel_crtc_destroy,
13022         .page_flip = drm_atomic_helper_page_flip,
13023         .atomic_duplicate_state = intel_crtc_duplicate_state,
13024         .atomic_destroy_state = intel_crtc_destroy_state,
13025         .set_crc_source = intel_crtc_set_crc_source,
13026         .verify_crc_source = intel_crtc_verify_crc_source,
13027         .get_crc_sources = intel_crtc_get_crc_sources,
13028 };
13029
13030 struct wait_rps_boost {
13031         struct wait_queue_entry wait;
13032
13033         struct drm_crtc *crtc;
13034         struct i915_request *request;
13035 };
13036
13037 static int do_rps_boost(struct wait_queue_entry *_wait,
13038                         unsigned mode, int sync, void *key)
13039 {
13040         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
13041         struct i915_request *rq = wait->request;
13042
13043         /*
13044          * If we missed the vblank, but the request is already running it
13045          * is reasonable to assume that it will complete before the next
13046          * vblank without our intervention, so leave RPS alone.
13047          */
13048         if (!i915_request_started(rq))
13049                 gen6_rps_boost(rq, NULL);
13050         i915_request_put(rq);
13051
13052         drm_crtc_vblank_put(wait->crtc);
13053
13054         list_del(&wait->wait.entry);
13055         kfree(wait);
13056         return 1;
13057 }
13058
13059 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
13060                                        struct dma_fence *fence)
13061 {
13062         struct wait_rps_boost *wait;
13063
13064         if (!dma_fence_is_i915(fence))
13065                 return;
13066
13067         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
13068                 return;
13069
13070         if (drm_crtc_vblank_get(crtc))
13071                 return;
13072
13073         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
13074         if (!wait) {
13075                 drm_crtc_vblank_put(crtc);
13076                 return;
13077         }
13078
13079         wait->request = to_request(dma_fence_get(fence));
13080         wait->crtc = crtc;
13081
13082         wait->wait.func = do_rps_boost;
13083         wait->wait.flags = 0;
13084
13085         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
13086 }
13087
13088 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
13089 {
13090         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
13091         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13092         struct drm_framebuffer *fb = plane_state->base.fb;
13093         struct i915_vma *vma;
13094
13095         if (plane->id == PLANE_CURSOR &&
13096             INTEL_INFO(dev_priv)->cursor_needs_physical) {
13097                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13098                 const int align = intel_cursor_alignment(dev_priv);
13099                 int err;
13100
13101                 err = i915_gem_object_attach_phys(obj, align);
13102                 if (err)
13103                         return err;
13104         }
13105
13106         vma = intel_pin_and_fence_fb_obj(fb,
13107                                          &plane_state->view,
13108                                          intel_plane_uses_fence(plane_state),
13109                                          &plane_state->flags);
13110         if (IS_ERR(vma))
13111                 return PTR_ERR(vma);
13112
13113         plane_state->vma = vma;
13114
13115         return 0;
13116 }
13117
13118 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
13119 {
13120         struct i915_vma *vma;
13121
13122         vma = fetch_and_zero(&old_plane_state->vma);
13123         if (vma)
13124                 intel_unpin_fb_vma(vma, old_plane_state->flags);
13125 }
13126
13127 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
13128 {
13129         struct i915_sched_attr attr = {
13130                 .priority = I915_PRIORITY_DISPLAY,
13131         };
13132
13133         i915_gem_object_wait_priority(obj, 0, &attr);
13134 }
13135
13136 /**
13137  * intel_prepare_plane_fb - Prepare fb for usage on plane
13138  * @plane: drm plane to prepare for
13139  * @new_state: the plane state being prepared
13140  *
13141  * Prepares a framebuffer for usage on a display plane.  Generally this
13142  * involves pinning the underlying object and updating the frontbuffer tracking
13143  * bits.  Some older platforms need special physical address handling for
13144  * cursor planes.
13145  *
13146  * Must be called with struct_mutex held.
13147  *
13148  * Returns 0 on success, negative error code on failure.
13149  */
13150 int
13151 intel_prepare_plane_fb(struct drm_plane *plane,
13152                        struct drm_plane_state *new_state)
13153 {
13154         struct intel_atomic_state *intel_state =
13155                 to_intel_atomic_state(new_state->state);
13156         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13157         struct drm_framebuffer *fb = new_state->fb;
13158         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13159         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
13160         int ret;
13161
13162         if (old_obj) {
13163                 struct drm_crtc_state *crtc_state =
13164                         drm_atomic_get_new_crtc_state(new_state->state,
13165                                                       plane->state->crtc);
13166
13167                 /* Big Hammer, we also need to ensure that any pending
13168                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
13169                  * current scanout is retired before unpinning the old
13170                  * framebuffer. Note that we rely on userspace rendering
13171                  * into the buffer attached to the pipe they are waiting
13172                  * on. If not, userspace generates a GPU hang with IPEHR
13173                  * point to the MI_WAIT_FOR_EVENT.
13174                  *
13175                  * This should only fail upon a hung GPU, in which case we
13176                  * can safely continue.
13177                  */
13178                 if (needs_modeset(crtc_state)) {
13179                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13180                                                               old_obj->resv, NULL,
13181                                                               false, 0,
13182                                                               GFP_KERNEL);
13183                         if (ret < 0)
13184                                 return ret;
13185                 }
13186         }
13187
13188         if (new_state->fence) { /* explicit fencing */
13189                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
13190                                                     new_state->fence,
13191                                                     I915_FENCE_TIMEOUT,
13192                                                     GFP_KERNEL);
13193                 if (ret < 0)
13194                         return ret;
13195         }
13196
13197         if (!obj)
13198                 return 0;
13199
13200         ret = i915_gem_object_pin_pages(obj);
13201         if (ret)
13202                 return ret;
13203
13204         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13205         if (ret) {
13206                 i915_gem_object_unpin_pages(obj);
13207                 return ret;
13208         }
13209
13210         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
13211
13212         mutex_unlock(&dev_priv->drm.struct_mutex);
13213         i915_gem_object_unpin_pages(obj);
13214         if (ret)
13215                 return ret;
13216
13217         fb_obj_bump_render_priority(obj);
13218         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
13219
13220         if (!new_state->fence) { /* implicit fencing */
13221                 struct dma_fence *fence;
13222
13223                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13224                                                       obj->resv, NULL,
13225                                                       false, I915_FENCE_TIMEOUT,
13226                                                       GFP_KERNEL);
13227                 if (ret < 0)
13228                         return ret;
13229
13230                 fence = reservation_object_get_excl_rcu(obj->resv);
13231                 if (fence) {
13232                         add_rps_boost_after_vblank(new_state->crtc, fence);
13233                         dma_fence_put(fence);
13234                 }
13235         } else {
13236                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
13237         }
13238
13239         /*
13240          * We declare pageflips to be interactive and so merit a small bias
13241          * towards upclocking to deliver the frame on time. By only changing
13242          * the RPS thresholds to sample more regularly and aim for higher
13243          * clocks we can hopefully deliver low power workloads (like kodi)
13244          * that are not quite steady state without resorting to forcing
13245          * maximum clocks following a vblank miss (see do_rps_boost()).
13246          */
13247         if (!intel_state->rps_interactive) {
13248                 intel_rps_mark_interactive(dev_priv, true);
13249                 intel_state->rps_interactive = true;
13250         }
13251
13252         return 0;
13253 }
13254
13255 /**
13256  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13257  * @plane: drm plane to clean up for
13258  * @old_state: the state from the previous modeset
13259  *
13260  * Cleans up a framebuffer that has just been removed from a plane.
13261  *
13262  * Must be called with struct_mutex held.
13263  */
13264 void
13265 intel_cleanup_plane_fb(struct drm_plane *plane,
13266                        struct drm_plane_state *old_state)
13267 {
13268         struct intel_atomic_state *intel_state =
13269                 to_intel_atomic_state(old_state->state);
13270         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13271
13272         if (intel_state->rps_interactive) {
13273                 intel_rps_mark_interactive(dev_priv, false);
13274                 intel_state->rps_interactive = false;
13275         }
13276
13277         /* Should only be called after a successful intel_prepare_plane_fb()! */
13278         mutex_lock(&dev_priv->drm.struct_mutex);
13279         intel_plane_unpin_fb(to_intel_plane_state(old_state));
13280         mutex_unlock(&dev_priv->drm.struct_mutex);
13281 }
13282
13283 int
13284 skl_max_scale(const struct intel_crtc_state *crtc_state,
13285               u32 pixel_format)
13286 {
13287         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13288         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13289         int max_scale, mult;
13290         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
13291
13292         if (!crtc_state->base.enable)
13293                 return DRM_PLANE_HELPER_NO_SCALING;
13294
13295         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13296         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
13297
13298         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
13299                 max_dotclk *= 2;
13300
13301         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
13302                 return DRM_PLANE_HELPER_NO_SCALING;
13303
13304         /*
13305          * skl max scale is lower of:
13306          *    close to 3 but not 3, -1 is for that purpose
13307          *            or
13308          *    cdclk/crtc_clock
13309          */
13310         mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
13311         tmpclk1 = (1 << 16) * mult - 1;
13312         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
13313         max_scale = min(tmpclk1, tmpclk2);
13314
13315         return max_scale;
13316 }
13317
13318 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13319                                     struct drm_crtc_state *old_crtc_state)
13320 {
13321         struct drm_device *dev = crtc->dev;
13322         struct drm_i915_private *dev_priv = to_i915(dev);
13323         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13324         struct intel_crtc_state *old_intel_cstate =
13325                 to_intel_crtc_state(old_crtc_state);
13326         struct intel_atomic_state *old_intel_state =
13327                 to_intel_atomic_state(old_crtc_state->state);
13328         struct intel_crtc_state *intel_cstate =
13329                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13330         bool modeset = needs_modeset(&intel_cstate->base);
13331
13332         if (!modeset &&
13333             (intel_cstate->base.color_mgmt_changed ||
13334              intel_cstate->update_pipe)) {
13335                 intel_color_set_csc(&intel_cstate->base);
13336                 intel_color_load_luts(&intel_cstate->base);
13337         }
13338
13339         /* Perform vblank evasion around commit operation */
13340         intel_pipe_update_start(intel_cstate);
13341
13342         if (modeset)
13343                 goto out;
13344
13345         if (intel_cstate->update_pipe)
13346                 intel_update_pipe_config(old_intel_cstate, intel_cstate);
13347         else if (INTEL_GEN(dev_priv) >= 9)
13348                 skl_detach_scalers(intel_crtc);
13349
13350 out:
13351         if (dev_priv->display.atomic_update_watermarks)
13352                 dev_priv->display.atomic_update_watermarks(old_intel_state,
13353                                                            intel_cstate);
13354 }
13355
13356 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
13357                                   struct intel_crtc_state *crtc_state)
13358 {
13359         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13360
13361         if (!IS_GEN2(dev_priv))
13362                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
13363
13364         if (crtc_state->has_pch_encoder) {
13365                 enum pipe pch_transcoder =
13366                         intel_crtc_pch_transcoder(crtc);
13367
13368                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
13369         }
13370 }
13371
13372 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13373                                      struct drm_crtc_state *old_crtc_state)
13374 {
13375         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13376         struct intel_atomic_state *old_intel_state =
13377                 to_intel_atomic_state(old_crtc_state->state);
13378         struct intel_crtc_state *new_crtc_state =
13379                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13380
13381         intel_pipe_update_end(new_crtc_state);
13382
13383         if (new_crtc_state->update_pipe &&
13384             !needs_modeset(&new_crtc_state->base) &&
13385             old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
13386                 intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
13387 }
13388
13389 /**
13390  * intel_plane_destroy - destroy a plane
13391  * @plane: plane to destroy
13392  *
13393  * Common destruction function for all types of planes (primary, cursor,
13394  * sprite).
13395  */
13396 void intel_plane_destroy(struct drm_plane *plane)
13397 {
13398         drm_plane_cleanup(plane);
13399         kfree(to_intel_plane(plane));
13400 }
13401
13402 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
13403                                             u32 format, u64 modifier)
13404 {
13405         switch (modifier) {
13406         case DRM_FORMAT_MOD_LINEAR:
13407         case I915_FORMAT_MOD_X_TILED:
13408                 break;
13409         default:
13410                 return false;
13411         }
13412
13413         switch (format) {
13414         case DRM_FORMAT_C8:
13415         case DRM_FORMAT_RGB565:
13416         case DRM_FORMAT_XRGB1555:
13417         case DRM_FORMAT_XRGB8888:
13418                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13419                         modifier == I915_FORMAT_MOD_X_TILED;
13420         default:
13421                 return false;
13422         }
13423 }
13424
13425 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
13426                                             u32 format, u64 modifier)
13427 {
13428         switch (modifier) {
13429         case DRM_FORMAT_MOD_LINEAR:
13430         case I915_FORMAT_MOD_X_TILED:
13431                 break;
13432         default:
13433                 return false;
13434         }
13435
13436         switch (format) {
13437         case DRM_FORMAT_C8:
13438         case DRM_FORMAT_RGB565:
13439         case DRM_FORMAT_XRGB8888:
13440         case DRM_FORMAT_XBGR8888:
13441         case DRM_FORMAT_XRGB2101010:
13442         case DRM_FORMAT_XBGR2101010:
13443                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13444                         modifier == I915_FORMAT_MOD_X_TILED;
13445         default:
13446                 return false;
13447         }
13448 }
13449
13450 static bool skl_plane_format_mod_supported(struct drm_plane *_plane,
13451                                            u32 format, u64 modifier)
13452 {
13453         struct intel_plane *plane = to_intel_plane(_plane);
13454
13455         switch (modifier) {
13456         case DRM_FORMAT_MOD_LINEAR:
13457         case I915_FORMAT_MOD_X_TILED:
13458         case I915_FORMAT_MOD_Y_TILED:
13459         case I915_FORMAT_MOD_Yf_TILED:
13460                 break;
13461         case I915_FORMAT_MOD_Y_TILED_CCS:
13462         case I915_FORMAT_MOD_Yf_TILED_CCS:
13463                 if (!plane->has_ccs)
13464                         return false;
13465                 break;
13466         default:
13467                 return false;
13468         }
13469
13470         switch (format) {
13471         case DRM_FORMAT_XRGB8888:
13472         case DRM_FORMAT_XBGR8888:
13473         case DRM_FORMAT_ARGB8888:
13474         case DRM_FORMAT_ABGR8888:
13475                 if (is_ccs_modifier(modifier))
13476                         return true;
13477                 /* fall through */
13478         case DRM_FORMAT_RGB565:
13479         case DRM_FORMAT_XRGB2101010:
13480         case DRM_FORMAT_XBGR2101010:
13481         case DRM_FORMAT_YUYV:
13482         case DRM_FORMAT_YVYU:
13483         case DRM_FORMAT_UYVY:
13484         case DRM_FORMAT_VYUY:
13485         case DRM_FORMAT_NV12:
13486                 if (modifier == I915_FORMAT_MOD_Yf_TILED)
13487                         return true;
13488                 /* fall through */
13489         case DRM_FORMAT_C8:
13490                 if (modifier == DRM_FORMAT_MOD_LINEAR ||
13491                     modifier == I915_FORMAT_MOD_X_TILED ||
13492                     modifier == I915_FORMAT_MOD_Y_TILED)
13493                         return true;
13494                 /* fall through */
13495         default:
13496                 return false;
13497         }
13498 }
13499
13500 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
13501                                               u32 format, u64 modifier)
13502 {
13503         return modifier == DRM_FORMAT_MOD_LINEAR &&
13504                 format == DRM_FORMAT_ARGB8888;
13505 }
13506
13507 static struct drm_plane_funcs skl_plane_funcs = {
13508         .update_plane = drm_atomic_helper_update_plane,
13509         .disable_plane = drm_atomic_helper_disable_plane,
13510         .destroy = intel_plane_destroy,
13511         .atomic_get_property = intel_plane_atomic_get_property,
13512         .atomic_set_property = intel_plane_atomic_set_property,
13513         .atomic_duplicate_state = intel_plane_duplicate_state,
13514         .atomic_destroy_state = intel_plane_destroy_state,
13515         .format_mod_supported = skl_plane_format_mod_supported,
13516 };
13517
13518 static struct drm_plane_funcs i965_plane_funcs = {
13519         .update_plane = drm_atomic_helper_update_plane,
13520         .disable_plane = drm_atomic_helper_disable_plane,
13521         .destroy = intel_plane_destroy,
13522         .atomic_get_property = intel_plane_atomic_get_property,
13523         .atomic_set_property = intel_plane_atomic_set_property,
13524         .atomic_duplicate_state = intel_plane_duplicate_state,
13525         .atomic_destroy_state = intel_plane_destroy_state,
13526         .format_mod_supported = i965_plane_format_mod_supported,
13527 };
13528
13529 static struct drm_plane_funcs i8xx_plane_funcs = {
13530         .update_plane = drm_atomic_helper_update_plane,
13531         .disable_plane = drm_atomic_helper_disable_plane,
13532         .destroy = intel_plane_destroy,
13533         .atomic_get_property = intel_plane_atomic_get_property,
13534         .atomic_set_property = intel_plane_atomic_set_property,
13535         .atomic_duplicate_state = intel_plane_duplicate_state,
13536         .atomic_destroy_state = intel_plane_destroy_state,
13537         .format_mod_supported = i8xx_plane_format_mod_supported,
13538 };
13539
13540 static int
13541 intel_legacy_cursor_update(struct drm_plane *plane,
13542                            struct drm_crtc *crtc,
13543                            struct drm_framebuffer *fb,
13544                            int crtc_x, int crtc_y,
13545                            unsigned int crtc_w, unsigned int crtc_h,
13546                            uint32_t src_x, uint32_t src_y,
13547                            uint32_t src_w, uint32_t src_h,
13548                            struct drm_modeset_acquire_ctx *ctx)
13549 {
13550         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
13551         int ret;
13552         struct drm_plane_state *old_plane_state, *new_plane_state;
13553         struct intel_plane *intel_plane = to_intel_plane(plane);
13554         struct drm_framebuffer *old_fb;
13555         struct intel_crtc_state *crtc_state =
13556                 to_intel_crtc_state(crtc->state);
13557         struct intel_crtc_state *new_crtc_state;
13558
13559         /*
13560          * When crtc is inactive or there is a modeset pending,
13561          * wait for it to complete in the slowpath
13562          */
13563         if (!crtc_state->base.active || needs_modeset(&crtc_state->base) ||
13564             crtc_state->update_pipe)
13565                 goto slow;
13566
13567         old_plane_state = plane->state;
13568         /*
13569          * Don't do an async update if there is an outstanding commit modifying
13570          * the plane.  This prevents our async update's changes from getting
13571          * overridden by a previous synchronous update's state.
13572          */
13573         if (old_plane_state->commit &&
13574             !try_wait_for_completion(&old_plane_state->commit->hw_done))
13575                 goto slow;
13576
13577         /*
13578          * If any parameters change that may affect watermarks,
13579          * take the slowpath. Only changing fb or position should be
13580          * in the fastpath.
13581          */
13582         if (old_plane_state->crtc != crtc ||
13583             old_plane_state->src_w != src_w ||
13584             old_plane_state->src_h != src_h ||
13585             old_plane_state->crtc_w != crtc_w ||
13586             old_plane_state->crtc_h != crtc_h ||
13587             !old_plane_state->fb != !fb)
13588                 goto slow;
13589
13590         new_plane_state = intel_plane_duplicate_state(plane);
13591         if (!new_plane_state)
13592                 return -ENOMEM;
13593
13594         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
13595         if (!new_crtc_state) {
13596                 ret = -ENOMEM;
13597                 goto out_free;
13598         }
13599
13600         drm_atomic_set_fb_for_plane(new_plane_state, fb);
13601
13602         new_plane_state->src_x = src_x;
13603         new_plane_state->src_y = src_y;
13604         new_plane_state->src_w = src_w;
13605         new_plane_state->src_h = src_h;
13606         new_plane_state->crtc_x = crtc_x;
13607         new_plane_state->crtc_y = crtc_y;
13608         new_plane_state->crtc_w = crtc_w;
13609         new_plane_state->crtc_h = crtc_h;
13610
13611         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
13612                                                   to_intel_plane_state(old_plane_state),
13613                                                   to_intel_plane_state(new_plane_state));
13614         if (ret)
13615                 goto out_free;
13616
13617         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13618         if (ret)
13619                 goto out_free;
13620
13621         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
13622         if (ret)
13623                 goto out_unlock;
13624
13625         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
13626
13627         old_fb = old_plane_state->fb;
13628         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
13629                           intel_plane->frontbuffer_bit);
13630
13631         /* Swap plane state */
13632         plane->state = new_plane_state;
13633
13634         /*
13635          * We cannot swap crtc_state as it may be in use by an atomic commit or
13636          * page flip that's running simultaneously. If we swap crtc_state and
13637          * destroy the old state, we will cause a use-after-free there.
13638          *
13639          * Only update active_planes, which is needed for our internal
13640          * bookkeeping. Either value will do the right thing when updating
13641          * planes atomically. If the cursor was part of the atomic update then
13642          * we would have taken the slowpath.
13643          */
13644         crtc_state->active_planes = new_crtc_state->active_planes;
13645
13646         if (plane->state->visible) {
13647                 trace_intel_update_plane(plane, to_intel_crtc(crtc));
13648                 intel_plane->update_plane(intel_plane, crtc_state,
13649                                           to_intel_plane_state(plane->state));
13650         } else {
13651                 trace_intel_disable_plane(plane, to_intel_crtc(crtc));
13652                 intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
13653         }
13654
13655         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
13656
13657 out_unlock:
13658         mutex_unlock(&dev_priv->drm.struct_mutex);
13659 out_free:
13660         if (new_crtc_state)
13661                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
13662         if (ret)
13663                 intel_plane_destroy_state(plane, new_plane_state);
13664         else
13665                 intel_plane_destroy_state(plane, old_plane_state);
13666         return ret;
13667
13668 slow:
13669         return drm_atomic_helper_update_plane(plane, crtc, fb,
13670                                               crtc_x, crtc_y, crtc_w, crtc_h,
13671                                               src_x, src_y, src_w, src_h, ctx);
13672 }
13673
13674 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
13675         .update_plane = intel_legacy_cursor_update,
13676         .disable_plane = drm_atomic_helper_disable_plane,
13677         .destroy = intel_plane_destroy,
13678         .atomic_get_property = intel_plane_atomic_get_property,
13679         .atomic_set_property = intel_plane_atomic_set_property,
13680         .atomic_duplicate_state = intel_plane_duplicate_state,
13681         .atomic_destroy_state = intel_plane_destroy_state,
13682         .format_mod_supported = intel_cursor_format_mod_supported,
13683 };
13684
13685 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
13686                                enum i9xx_plane_id i9xx_plane)
13687 {
13688         if (!HAS_FBC(dev_priv))
13689                 return false;
13690
13691         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
13692                 return i9xx_plane == PLANE_A; /* tied to pipe A */
13693         else if (IS_IVYBRIDGE(dev_priv))
13694                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
13695                         i9xx_plane == PLANE_C;
13696         else if (INTEL_GEN(dev_priv) >= 4)
13697                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
13698         else
13699                 return i9xx_plane == PLANE_A;
13700 }
13701
13702 static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv,
13703                               enum pipe pipe, enum plane_id plane_id)
13704 {
13705         if (!HAS_FBC(dev_priv))
13706                 return false;
13707
13708         return pipe == PIPE_A && plane_id == PLANE_PRIMARY;
13709 }
13710
13711 bool skl_plane_has_planar(struct drm_i915_private *dev_priv,
13712                           enum pipe pipe, enum plane_id plane_id)
13713 {
13714         /*
13715          * FIXME: ICL requires two hardware planes for scanning out NV12
13716          * framebuffers. Do not advertize support until this is implemented.
13717          */
13718         if (INTEL_GEN(dev_priv) >= 11)
13719                 return false;
13720
13721         if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))
13722                 return false;
13723
13724         if (INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv) && pipe == PIPE_C)
13725                 return false;
13726
13727         if (plane_id != PLANE_PRIMARY && plane_id != PLANE_SPRITE0)
13728                 return false;
13729
13730         return true;
13731 }
13732
13733 static struct intel_plane *
13734 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13735 {
13736         struct intel_plane *primary = NULL;
13737         struct intel_plane_state *state = NULL;
13738         const struct drm_plane_funcs *plane_funcs;
13739         const uint32_t *intel_primary_formats;
13740         unsigned int supported_rotations;
13741         unsigned int num_formats;
13742         const uint64_t *modifiers;
13743         int ret;
13744
13745         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
13746         if (!primary) {
13747                 ret = -ENOMEM;
13748                 goto fail;
13749         }
13750
13751         state = intel_create_plane_state(&primary->base);
13752         if (!state) {
13753                 ret = -ENOMEM;
13754                 goto fail;
13755         }
13756
13757         primary->base.state = &state->base;
13758
13759         if (INTEL_GEN(dev_priv) >= 9)
13760                 state->scaler_id = -1;
13761         primary->pipe = pipe;
13762         /*
13763          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
13764          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
13765          */
13766         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
13767                 primary->i9xx_plane = (enum i9xx_plane_id) !pipe;
13768         else
13769                 primary->i9xx_plane = (enum i9xx_plane_id) pipe;
13770         primary->id = PLANE_PRIMARY;
13771         primary->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, primary->id);
13772
13773         if (INTEL_GEN(dev_priv) >= 9)
13774                 primary->has_fbc = skl_plane_has_fbc(dev_priv,
13775                                                      primary->pipe,
13776                                                      primary->id);
13777         else
13778                 primary->has_fbc = i9xx_plane_has_fbc(dev_priv,
13779                                                       primary->i9xx_plane);
13780
13781         if (primary->has_fbc) {
13782                 struct intel_fbc *fbc = &dev_priv->fbc;
13783
13784                 fbc->possible_framebuffer_bits |= primary->frontbuffer_bit;
13785         }
13786
13787         if (INTEL_GEN(dev_priv) >= 9) {
13788                 primary->has_ccs = skl_plane_has_ccs(dev_priv, pipe,
13789                                                      PLANE_PRIMARY);
13790
13791                 if (skl_plane_has_planar(dev_priv, pipe, PLANE_PRIMARY)) {
13792                         intel_primary_formats = skl_pri_planar_formats;
13793                         num_formats = ARRAY_SIZE(skl_pri_planar_formats);
13794                 } else {
13795                         intel_primary_formats = skl_primary_formats;
13796                         num_formats = ARRAY_SIZE(skl_primary_formats);
13797                 }
13798
13799                 if (primary->has_ccs)
13800                         modifiers = skl_format_modifiers_ccs;
13801                 else
13802                         modifiers = skl_format_modifiers_noccs;
13803
13804                 primary->max_stride = skl_plane_max_stride;
13805                 primary->update_plane = skl_update_plane;
13806                 primary->disable_plane = skl_disable_plane;
13807                 primary->get_hw_state = skl_plane_get_hw_state;
13808                 primary->check_plane = skl_plane_check;
13809
13810                 plane_funcs = &skl_plane_funcs;
13811         } else if (INTEL_GEN(dev_priv) >= 4) {
13812                 intel_primary_formats = i965_primary_formats;
13813                 num_formats = ARRAY_SIZE(i965_primary_formats);
13814                 modifiers = i9xx_format_modifiers;
13815
13816                 primary->max_stride = i9xx_plane_max_stride;
13817                 primary->update_plane = i9xx_update_plane;
13818                 primary->disable_plane = i9xx_disable_plane;
13819                 primary->get_hw_state = i9xx_plane_get_hw_state;
13820                 primary->check_plane = i9xx_plane_check;
13821
13822                 plane_funcs = &i965_plane_funcs;
13823         } else {
13824                 intel_primary_formats = i8xx_primary_formats;
13825                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13826                 modifiers = i9xx_format_modifiers;
13827
13828                 primary->max_stride = i9xx_plane_max_stride;
13829                 primary->update_plane = i9xx_update_plane;
13830                 primary->disable_plane = i9xx_disable_plane;
13831                 primary->get_hw_state = i9xx_plane_get_hw_state;
13832                 primary->check_plane = i9xx_plane_check;
13833
13834                 plane_funcs = &i8xx_plane_funcs;
13835         }
13836
13837         if (INTEL_GEN(dev_priv) >= 9)
13838                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13839                                                0, plane_funcs,
13840                                                intel_primary_formats, num_formats,
13841                                                modifiers,
13842                                                DRM_PLANE_TYPE_PRIMARY,
13843                                                "plane 1%c", pipe_name(pipe));
13844         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
13845                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13846                                                0, plane_funcs,
13847                                                intel_primary_formats, num_formats,
13848                                                modifiers,
13849                                                DRM_PLANE_TYPE_PRIMARY,
13850                                                "primary %c", pipe_name(pipe));
13851         else
13852                 ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
13853                                                0, plane_funcs,
13854                                                intel_primary_formats, num_formats,
13855                                                modifiers,
13856                                                DRM_PLANE_TYPE_PRIMARY,
13857                                                "plane %c",
13858                                                plane_name(primary->i9xx_plane));
13859         if (ret)
13860                 goto fail;
13861
13862         if (INTEL_GEN(dev_priv) >= 10) {
13863                 supported_rotations =
13864                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
13865                         DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
13866                         DRM_MODE_REFLECT_X;
13867         } else if (INTEL_GEN(dev_priv) >= 9) {
13868                 supported_rotations =
13869                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
13870                         DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270;
13871         } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
13872                 supported_rotations =
13873                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
13874                         DRM_MODE_REFLECT_X;
13875         } else if (INTEL_GEN(dev_priv) >= 4) {
13876                 supported_rotations =
13877                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
13878         } else {
13879                 supported_rotations = DRM_MODE_ROTATE_0;
13880         }
13881
13882         if (INTEL_GEN(dev_priv) >= 4)
13883                 drm_plane_create_rotation_property(&primary->base,
13884                                                    DRM_MODE_ROTATE_0,
13885                                                    supported_rotations);
13886
13887         if (INTEL_GEN(dev_priv) >= 9) {
13888                 drm_plane_create_color_properties(&primary->base,
13889                                                   BIT(DRM_COLOR_YCBCR_BT601) |
13890                                                   BIT(DRM_COLOR_YCBCR_BT709),
13891                                                   BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) |
13892                                                   BIT(DRM_COLOR_YCBCR_FULL_RANGE),
13893                                                   DRM_COLOR_YCBCR_BT709,
13894                                                   DRM_COLOR_YCBCR_LIMITED_RANGE);
13895
13896                 drm_plane_create_alpha_property(&primary->base);
13897                 drm_plane_create_blend_mode_property(&primary->base,
13898                                                      BIT(DRM_MODE_BLEND_PIXEL_NONE) |
13899                                                      BIT(DRM_MODE_BLEND_PREMULTI) |
13900                                                      BIT(DRM_MODE_BLEND_COVERAGE));
13901         }
13902
13903         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
13904
13905         return primary;
13906
13907 fail:
13908         kfree(state);
13909         kfree(primary);
13910
13911         return ERR_PTR(ret);
13912 }
13913
13914 static struct intel_plane *
13915 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
13916                           enum pipe pipe)
13917 {
13918         struct intel_plane *cursor = NULL;
13919         struct intel_plane_state *state = NULL;
13920         int ret;
13921
13922         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
13923         if (!cursor) {
13924                 ret = -ENOMEM;
13925                 goto fail;
13926         }
13927
13928         state = intel_create_plane_state(&cursor->base);
13929         if (!state) {
13930                 ret = -ENOMEM;
13931                 goto fail;
13932         }
13933
13934         cursor->base.state = &state->base;
13935
13936         cursor->pipe = pipe;
13937         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
13938         cursor->id = PLANE_CURSOR;
13939         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
13940
13941         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
13942                 cursor->max_stride = i845_cursor_max_stride;
13943                 cursor->update_plane = i845_update_cursor;
13944                 cursor->disable_plane = i845_disable_cursor;
13945                 cursor->get_hw_state = i845_cursor_get_hw_state;
13946                 cursor->check_plane = i845_check_cursor;
13947         } else {
13948                 cursor->max_stride = i9xx_cursor_max_stride;
13949                 cursor->update_plane = i9xx_update_cursor;
13950                 cursor->disable_plane = i9xx_disable_cursor;
13951                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
13952                 cursor->check_plane = i9xx_check_cursor;
13953         }
13954
13955         cursor->cursor.base = ~0;
13956         cursor->cursor.cntl = ~0;
13957
13958         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
13959                 cursor->cursor.size = ~0;
13960
13961         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
13962                                        0, &intel_cursor_plane_funcs,
13963                                        intel_cursor_formats,
13964                                        ARRAY_SIZE(intel_cursor_formats),
13965                                        cursor_format_modifiers,
13966                                        DRM_PLANE_TYPE_CURSOR,
13967                                        "cursor %c", pipe_name(pipe));
13968         if (ret)
13969                 goto fail;
13970
13971         if (INTEL_GEN(dev_priv) >= 4)
13972                 drm_plane_create_rotation_property(&cursor->base,
13973                                                    DRM_MODE_ROTATE_0,
13974                                                    DRM_MODE_ROTATE_0 |
13975                                                    DRM_MODE_ROTATE_180);
13976
13977         if (INTEL_GEN(dev_priv) >= 9)
13978                 state->scaler_id = -1;
13979
13980         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
13981
13982         return cursor;
13983
13984 fail:
13985         kfree(state);
13986         kfree(cursor);
13987
13988         return ERR_PTR(ret);
13989 }
13990
13991 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
13992                                     struct intel_crtc_state *crtc_state)
13993 {
13994         struct intel_crtc_scaler_state *scaler_state =
13995                 &crtc_state->scaler_state;
13996         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13997         int i;
13998
13999         crtc->num_scalers = dev_priv->info.num_scalers[crtc->pipe];
14000         if (!crtc->num_scalers)
14001                 return;
14002
14003         for (i = 0; i < crtc->num_scalers; i++) {
14004                 struct intel_scaler *scaler = &scaler_state->scalers[i];
14005
14006                 scaler->in_use = 0;
14007                 scaler->mode = 0;
14008         }
14009
14010         scaler_state->scaler_id = -1;
14011 }
14012
14013 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
14014 {
14015         struct intel_crtc *intel_crtc;
14016         struct intel_crtc_state *crtc_state = NULL;
14017         struct intel_plane *primary = NULL;
14018         struct intel_plane *cursor = NULL;
14019         int sprite, ret;
14020
14021         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
14022         if (!intel_crtc)
14023                 return -ENOMEM;
14024
14025         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
14026         if (!crtc_state) {
14027                 ret = -ENOMEM;
14028                 goto fail;
14029         }
14030         intel_crtc->config = crtc_state;
14031         intel_crtc->base.state = &crtc_state->base;
14032         crtc_state->base.crtc = &intel_crtc->base;
14033
14034         primary = intel_primary_plane_create(dev_priv, pipe);
14035         if (IS_ERR(primary)) {
14036                 ret = PTR_ERR(primary);
14037                 goto fail;
14038         }
14039         intel_crtc->plane_ids_mask |= BIT(primary->id);
14040
14041         for_each_sprite(dev_priv, pipe, sprite) {
14042                 struct intel_plane *plane;
14043
14044                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
14045                 if (IS_ERR(plane)) {
14046                         ret = PTR_ERR(plane);
14047                         goto fail;
14048                 }
14049                 intel_crtc->plane_ids_mask |= BIT(plane->id);
14050         }
14051
14052         cursor = intel_cursor_plane_create(dev_priv, pipe);
14053         if (IS_ERR(cursor)) {
14054                 ret = PTR_ERR(cursor);
14055                 goto fail;
14056         }
14057         intel_crtc->plane_ids_mask |= BIT(cursor->id);
14058
14059         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
14060                                         &primary->base, &cursor->base,
14061                                         &intel_crtc_funcs,
14062                                         "pipe %c", pipe_name(pipe));
14063         if (ret)
14064                 goto fail;
14065
14066         intel_crtc->pipe = pipe;
14067
14068         /* initialize shared scalers */
14069         intel_crtc_init_scalers(intel_crtc, crtc_state);
14070
14071         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
14072                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
14073         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
14074
14075         if (INTEL_GEN(dev_priv) < 9) {
14076                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
14077
14078                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14079                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
14080                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
14081         }
14082
14083         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14084
14085         intel_color_init(&intel_crtc->base);
14086
14087         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14088
14089         return 0;
14090
14091 fail:
14092         /*
14093          * drm_mode_config_cleanup() will free up any
14094          * crtcs/planes already initialized.
14095          */
14096         kfree(crtc_state);
14097         kfree(intel_crtc);
14098
14099         return ret;
14100 }
14101
14102 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
14103 {
14104         struct drm_device *dev = connector->base.dev;
14105
14106         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
14107
14108         if (!connector->base.state->crtc)
14109                 return INVALID_PIPE;
14110
14111         return to_intel_crtc(connector->base.state->crtc)->pipe;
14112 }
14113
14114 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
14115                                       struct drm_file *file)
14116 {
14117         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14118         struct drm_crtc *drmmode_crtc;
14119         struct intel_crtc *crtc;
14120
14121         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
14122         if (!drmmode_crtc)
14123                 return -ENOENT;
14124
14125         crtc = to_intel_crtc(drmmode_crtc);
14126         pipe_from_crtc_id->pipe = crtc->pipe;
14127
14128         return 0;
14129 }
14130
14131 static int intel_encoder_clones(struct intel_encoder *encoder)
14132 {
14133         struct drm_device *dev = encoder->base.dev;
14134         struct intel_encoder *source_encoder;
14135         int index_mask = 0;
14136         int entry = 0;
14137
14138         for_each_intel_encoder(dev, source_encoder) {
14139                 if (encoders_cloneable(encoder, source_encoder))
14140                         index_mask |= (1 << entry);
14141
14142                 entry++;
14143         }
14144
14145         return index_mask;
14146 }
14147
14148 static bool has_edp_a(struct drm_i915_private *dev_priv)
14149 {
14150         if (!IS_MOBILE(dev_priv))
14151                 return false;
14152
14153         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14154                 return false;
14155
14156         if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14157                 return false;
14158
14159         return true;
14160 }
14161
14162 static bool intel_crt_present(struct drm_i915_private *dev_priv)
14163 {
14164         if (INTEL_GEN(dev_priv) >= 9)
14165                 return false;
14166
14167         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
14168                 return false;
14169
14170         if (IS_CHERRYVIEW(dev_priv))
14171                 return false;
14172
14173         if (HAS_PCH_LPT_H(dev_priv) &&
14174             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
14175                 return false;
14176
14177         /* DDI E can't be used if DDI A requires 4 lanes */
14178         if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
14179                 return false;
14180
14181         if (!dev_priv->vbt.int_crt_support)
14182                 return false;
14183
14184         return true;
14185 }
14186
14187 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
14188 {
14189         int pps_num;
14190         int pps_idx;
14191
14192         if (HAS_DDI(dev_priv))
14193                 return;
14194         /*
14195          * This w/a is needed at least on CPT/PPT, but to be sure apply it
14196          * everywhere where registers can be write protected.
14197          */
14198         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14199                 pps_num = 2;
14200         else
14201                 pps_num = 1;
14202
14203         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
14204                 u32 val = I915_READ(PP_CONTROL(pps_idx));
14205
14206                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
14207                 I915_WRITE(PP_CONTROL(pps_idx), val);
14208         }
14209 }
14210
14211 static void intel_pps_init(struct drm_i915_private *dev_priv)
14212 {
14213         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
14214                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
14215         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14216                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
14217         else
14218                 dev_priv->pps_mmio_base = PPS_BASE;
14219
14220         intel_pps_unlock_regs_wa(dev_priv);
14221 }
14222
14223 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
14224 {
14225         struct intel_encoder *encoder;
14226         bool dpd_is_edp = false;
14227
14228         intel_pps_init(dev_priv);
14229
14230         if (INTEL_INFO(dev_priv)->num_pipes == 0)
14231                 return;
14232
14233         /*
14234          * intel_edp_init_connector() depends on this completing first, to
14235          * prevent the registeration of both eDP and LVDS and the incorrect
14236          * sharing of the PPS.
14237          */
14238         intel_lvds_init(dev_priv);
14239
14240         if (intel_crt_present(dev_priv))
14241                 intel_crt_init(dev_priv);
14242
14243         if (IS_ICELAKE(dev_priv)) {
14244                 intel_ddi_init(dev_priv, PORT_A);
14245                 intel_ddi_init(dev_priv, PORT_B);
14246                 intel_ddi_init(dev_priv, PORT_C);
14247                 intel_ddi_init(dev_priv, PORT_D);
14248                 intel_ddi_init(dev_priv, PORT_E);
14249                 intel_ddi_init(dev_priv, PORT_F);
14250         } else if (IS_GEN9_LP(dev_priv)) {
14251                 /*
14252                  * FIXME: Broxton doesn't support port detection via the
14253                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14254                  * detect the ports.
14255                  */
14256                 intel_ddi_init(dev_priv, PORT_A);
14257                 intel_ddi_init(dev_priv, PORT_B);
14258                 intel_ddi_init(dev_priv, PORT_C);
14259
14260                 vlv_dsi_init(dev_priv);
14261         } else if (HAS_DDI(dev_priv)) {
14262                 int found;
14263
14264                 /*
14265                  * Haswell uses DDI functions to detect digital outputs.
14266                  * On SKL pre-D0 the strap isn't connected, so we assume
14267                  * it's there.
14268                  */
14269                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14270                 /* WaIgnoreDDIAStrap: skl */
14271                 if (found || IS_GEN9_BC(dev_priv))
14272                         intel_ddi_init(dev_priv, PORT_A);
14273
14274                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
14275                  * register */
14276                 found = I915_READ(SFUSE_STRAP);
14277
14278                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14279                         intel_ddi_init(dev_priv, PORT_B);
14280                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14281                         intel_ddi_init(dev_priv, PORT_C);
14282                 if (found & SFUSE_STRAP_DDID_DETECTED)
14283                         intel_ddi_init(dev_priv, PORT_D);
14284                 if (found & SFUSE_STRAP_DDIF_DETECTED)
14285                         intel_ddi_init(dev_priv, PORT_F);
14286                 /*
14287                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14288                  */
14289                 if (IS_GEN9_BC(dev_priv) &&
14290                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14291                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14292                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14293                         intel_ddi_init(dev_priv, PORT_E);
14294
14295         } else if (HAS_PCH_SPLIT(dev_priv)) {
14296                 int found;
14297                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
14298
14299                 if (has_edp_a(dev_priv))
14300                         intel_dp_init(dev_priv, DP_A, PORT_A);
14301
14302                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14303                         /* PCH SDVOB multiplex with HDMIB */
14304                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
14305                         if (!found)
14306                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
14307                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14308                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
14309                 }
14310
14311                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14312                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
14313
14314                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14315                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
14316
14317                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14318                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
14319
14320                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14321                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
14322         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
14323                 bool has_edp, has_port;
14324
14325                 /*
14326                  * The DP_DETECTED bit is the latched state of the DDC
14327                  * SDA pin at boot. However since eDP doesn't require DDC
14328                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14329                  * eDP ports may have been muxed to an alternate function.
14330                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14331                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14332                  * detect eDP ports.
14333                  *
14334                  * Sadly the straps seem to be missing sometimes even for HDMI
14335                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14336                  * and VBT for the presence of the port. Additionally we can't
14337                  * trust the port type the VBT declares as we've seen at least
14338                  * HDMI ports that the VBT claim are DP or eDP.
14339                  */
14340                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
14341                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14342                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
14343                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
14344                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14345                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
14346
14347                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
14348                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14349                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
14350                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
14351                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14352                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
14353
14354                 if (IS_CHERRYVIEW(dev_priv)) {
14355                         /*
14356                          * eDP not supported on port D,
14357                          * so no need to worry about it
14358                          */
14359                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14360                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
14361                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
14362                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
14363                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
14364                 }
14365
14366                 vlv_dsi_init(dev_priv);
14367         } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
14368                 bool found = false;
14369
14370                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14371                         DRM_DEBUG_KMS("probing SDVOB\n");
14372                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
14373                         if (!found && IS_G4X(dev_priv)) {
14374                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14375                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
14376                         }
14377
14378                         if (!found && IS_G4X(dev_priv))
14379                                 intel_dp_init(dev_priv, DP_B, PORT_B);
14380                 }
14381
14382                 /* Before G4X SDVOC doesn't have its own detect register */
14383
14384                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14385                         DRM_DEBUG_KMS("probing SDVOC\n");
14386                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
14387                 }
14388
14389                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14390
14391                         if (IS_G4X(dev_priv)) {
14392                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14393                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
14394                         }
14395                         if (IS_G4X(dev_priv))
14396                                 intel_dp_init(dev_priv, DP_C, PORT_C);
14397                 }
14398
14399                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
14400                         intel_dp_init(dev_priv, DP_D, PORT_D);
14401         } else if (IS_GEN2(dev_priv))
14402                 intel_dvo_init(dev_priv);
14403
14404         if (SUPPORTS_TV(dev_priv))
14405                 intel_tv_init(dev_priv);
14406
14407         intel_psr_init(dev_priv);
14408
14409         for_each_intel_encoder(&dev_priv->drm, encoder) {
14410                 encoder->base.possible_crtcs = encoder->crtc_mask;
14411                 encoder->base.possible_clones =
14412                         intel_encoder_clones(encoder);
14413         }
14414
14415         intel_init_pch_refclk(dev_priv);
14416
14417         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
14418 }
14419
14420 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14421 {
14422         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14423         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14424
14425         drm_framebuffer_cleanup(fb);
14426
14427         i915_gem_object_lock(obj);
14428         WARN_ON(!obj->framebuffer_references--);
14429         i915_gem_object_unlock(obj);
14430
14431         i915_gem_object_put(obj);
14432
14433         kfree(intel_fb);
14434 }
14435
14436 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14437                                                 struct drm_file *file,
14438                                                 unsigned int *handle)
14439 {
14440         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14441
14442         if (obj->userptr.mm) {
14443                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14444                 return -EINVAL;
14445         }
14446
14447         return drm_gem_handle_create(file, &obj->base, handle);
14448 }
14449
14450 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14451                                         struct drm_file *file,
14452                                         unsigned flags, unsigned color,
14453                                         struct drm_clip_rect *clips,
14454                                         unsigned num_clips)
14455 {
14456         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14457
14458         i915_gem_object_flush_if_display(obj);
14459         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14460
14461         return 0;
14462 }
14463
14464 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14465         .destroy = intel_user_framebuffer_destroy,
14466         .create_handle = intel_user_framebuffer_create_handle,
14467         .dirty = intel_user_framebuffer_dirty,
14468 };
14469
14470 static
14471 u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
14472                          uint64_t fb_modifier, uint32_t pixel_format)
14473 {
14474         struct intel_crtc *crtc;
14475         struct intel_plane *plane;
14476
14477         /*
14478          * We assume the primary plane for pipe A has
14479          * the highest stride limits of them all.
14480          */
14481         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
14482         plane = to_intel_plane(crtc->base.primary);
14483
14484         return plane->max_stride(plane, pixel_format, fb_modifier,
14485                                  DRM_MODE_ROTATE_0);
14486 }
14487
14488 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
14489                                   struct drm_i915_gem_object *obj,
14490                                   struct drm_mode_fb_cmd2 *mode_cmd)
14491 {
14492         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
14493         struct drm_framebuffer *fb = &intel_fb->base;
14494         struct drm_format_name_buf format_name;
14495         u32 pitch_limit;
14496         unsigned int tiling, stride;
14497         int ret = -EINVAL;
14498         int i;
14499
14500         i915_gem_object_lock(obj);
14501         obj->framebuffer_references++;
14502         tiling = i915_gem_object_get_tiling(obj);
14503         stride = i915_gem_object_get_stride(obj);
14504         i915_gem_object_unlock(obj);
14505
14506         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14507                 /*
14508                  * If there's a fence, enforce that
14509                  * the fb modifier and tiling mode match.
14510                  */
14511                 if (tiling != I915_TILING_NONE &&
14512                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14513                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
14514                         goto err;
14515                 }
14516         } else {
14517                 if (tiling == I915_TILING_X) {
14518                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14519                 } else if (tiling == I915_TILING_Y) {
14520                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
14521                         goto err;
14522                 }
14523         }
14524
14525         /* Passed in modifier sanity checking. */
14526         switch (mode_cmd->modifier[0]) {
14527         case I915_FORMAT_MOD_Y_TILED_CCS:
14528         case I915_FORMAT_MOD_Yf_TILED_CCS:
14529                 switch (mode_cmd->pixel_format) {
14530                 case DRM_FORMAT_XBGR8888:
14531                 case DRM_FORMAT_ABGR8888:
14532                 case DRM_FORMAT_XRGB8888:
14533                 case DRM_FORMAT_ARGB8888:
14534                         break;
14535                 default:
14536                         DRM_DEBUG_KMS("RC supported only with RGB8888 formats\n");
14537                         goto err;
14538                 }
14539                 /* fall through */
14540         case I915_FORMAT_MOD_Y_TILED:
14541         case I915_FORMAT_MOD_Yf_TILED:
14542                 if (INTEL_GEN(dev_priv) < 9) {
14543                         DRM_DEBUG_KMS("Unsupported tiling 0x%llx!\n",
14544                                       mode_cmd->modifier[0]);
14545                         goto err;
14546                 }
14547         case DRM_FORMAT_MOD_LINEAR:
14548         case I915_FORMAT_MOD_X_TILED:
14549                 break;
14550         default:
14551                 DRM_DEBUG_KMS("Unsupported fb modifier 0x%llx!\n",
14552                               mode_cmd->modifier[0]);
14553                 goto err;
14554         }
14555
14556         /*
14557          * gen2/3 display engine uses the fence if present,
14558          * so the tiling mode must match the fb modifier exactly.
14559          */
14560         if (INTEL_GEN(dev_priv) < 4 &&
14561             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14562                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
14563                 goto err;
14564         }
14565
14566         pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->modifier[0],
14567                                            mode_cmd->pixel_format);
14568         if (mode_cmd->pitches[0] > pitch_limit) {
14569                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
14570                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
14571                               "tiled" : "linear",
14572                               mode_cmd->pitches[0], pitch_limit);
14573                 goto err;
14574         }
14575
14576         /*
14577          * If there's a fence, enforce that
14578          * the fb pitch and fence stride match.
14579          */
14580         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
14581                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
14582                               mode_cmd->pitches[0], stride);
14583                 goto err;
14584         }
14585
14586         /* Reject formats not supported by any plane early. */
14587         switch (mode_cmd->pixel_format) {
14588         case DRM_FORMAT_C8:
14589         case DRM_FORMAT_RGB565:
14590         case DRM_FORMAT_XRGB8888:
14591         case DRM_FORMAT_ARGB8888:
14592                 break;
14593         case DRM_FORMAT_XRGB1555:
14594                 if (INTEL_GEN(dev_priv) > 3) {
14595                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14596                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14597                         goto err;
14598                 }
14599                 break;
14600         case DRM_FORMAT_ABGR8888:
14601                 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
14602                     INTEL_GEN(dev_priv) < 9) {
14603                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14604                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14605                         goto err;
14606                 }
14607                 break;
14608         case DRM_FORMAT_XBGR8888:
14609         case DRM_FORMAT_XRGB2101010:
14610         case DRM_FORMAT_XBGR2101010:
14611                 if (INTEL_GEN(dev_priv) < 4) {
14612                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14613                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14614                         goto err;
14615                 }
14616                 break;
14617         case DRM_FORMAT_ABGR2101010:
14618                 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
14619                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14620                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14621                         goto err;
14622                 }
14623                 break;
14624         case DRM_FORMAT_YUYV:
14625         case DRM_FORMAT_UYVY:
14626         case DRM_FORMAT_YVYU:
14627         case DRM_FORMAT_VYUY:
14628                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
14629                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14630                                       drm_get_format_name(mode_cmd->pixel_format, &format_name));
14631                         goto err;
14632                 }
14633                 break;
14634         case DRM_FORMAT_NV12:
14635                 if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) ||
14636                     IS_BROXTON(dev_priv) || INTEL_GEN(dev_priv) >= 11) {
14637                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14638                                       drm_get_format_name(mode_cmd->pixel_format,
14639                                                           &format_name));
14640                         goto err;
14641                 }
14642                 break;
14643         default:
14644                 DRM_DEBUG_KMS("unsupported pixel format: %s\n",
14645                               drm_get_format_name(mode_cmd->pixel_format, &format_name));
14646                 goto err;
14647         }
14648
14649         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14650         if (mode_cmd->offsets[0] != 0)
14651                 goto err;
14652
14653         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
14654
14655         if (fb->format->format == DRM_FORMAT_NV12 &&
14656             (fb->width < SKL_MIN_YUV_420_SRC_W ||
14657              fb->height < SKL_MIN_YUV_420_SRC_H ||
14658              (fb->width % 4) != 0 || (fb->height % 4) != 0)) {
14659                 DRM_DEBUG_KMS("src dimensions not correct for NV12\n");
14660                 return -EINVAL;
14661         }
14662
14663         for (i = 0; i < fb->format->num_planes; i++) {
14664                 u32 stride_alignment;
14665
14666                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
14667                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
14668                         goto err;
14669                 }
14670
14671                 stride_alignment = intel_fb_stride_alignment(fb, i);
14672
14673                 /*
14674                  * Display WA #0531: skl,bxt,kbl,glk
14675                  *
14676                  * Render decompression and plane width > 3840
14677                  * combined with horizontal panning requires the
14678                  * plane stride to be a multiple of 4. We'll just
14679                  * require the entire fb to accommodate that to avoid
14680                  * potential runtime errors at plane configuration time.
14681                  */
14682                 if (IS_GEN9(dev_priv) && i == 0 && fb->width > 3840 &&
14683                     is_ccs_modifier(fb->modifier))
14684                         stride_alignment *= 4;
14685
14686                 if (fb->pitches[i] & (stride_alignment - 1)) {
14687                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
14688                                       i, fb->pitches[i], stride_alignment);
14689                         goto err;
14690                 }
14691
14692                 fb->obj[i] = &obj->base;
14693         }
14694
14695         ret = intel_fill_fb_info(dev_priv, fb);
14696         if (ret)
14697                 goto err;
14698
14699         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
14700         if (ret) {
14701                 DRM_ERROR("framebuffer init failed %d\n", ret);
14702                 goto err;
14703         }
14704
14705         return 0;
14706
14707 err:
14708         i915_gem_object_lock(obj);
14709         obj->framebuffer_references--;
14710         i915_gem_object_unlock(obj);
14711         return ret;
14712 }
14713
14714 static struct drm_framebuffer *
14715 intel_user_framebuffer_create(struct drm_device *dev,
14716                               struct drm_file *filp,
14717                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
14718 {
14719         struct drm_framebuffer *fb;
14720         struct drm_i915_gem_object *obj;
14721         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14722
14723         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
14724         if (!obj)
14725                 return ERR_PTR(-ENOENT);
14726
14727         fb = intel_framebuffer_create(obj, &mode_cmd);
14728         if (IS_ERR(fb))
14729                 i915_gem_object_put(obj);
14730
14731         return fb;
14732 }
14733
14734 static void intel_atomic_state_free(struct drm_atomic_state *state)
14735 {
14736         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14737
14738         drm_atomic_state_default_release(state);
14739
14740         i915_sw_fence_fini(&intel_state->commit_ready);
14741
14742         kfree(state);
14743 }
14744
14745 static enum drm_mode_status
14746 intel_mode_valid(struct drm_device *dev,
14747                  const struct drm_display_mode *mode)
14748 {
14749         struct drm_i915_private *dev_priv = to_i915(dev);
14750         int hdisplay_max, htotal_max;
14751         int vdisplay_max, vtotal_max;
14752
14753         /*
14754          * Can't reject DBLSCAN here because Xorg ddxen can add piles
14755          * of DBLSCAN modes to the output's mode list when they detect
14756          * the scaling mode property on the connector. And they don't
14757          * ask the kernel to validate those modes in any way until
14758          * modeset time at which point the client gets a protocol error.
14759          * So in order to not upset those clients we silently ignore the
14760          * DBLSCAN flag on such connectors. For other connectors we will
14761          * reject modes with the DBLSCAN flag in encoder->compute_config().
14762          * And we always reject DBLSCAN modes in connector->mode_valid()
14763          * as we never want such modes on the connector's mode list.
14764          */
14765
14766         if (mode->vscan > 1)
14767                 return MODE_NO_VSCAN;
14768
14769         if (mode->flags & DRM_MODE_FLAG_HSKEW)
14770                 return MODE_H_ILLEGAL;
14771
14772         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
14773                            DRM_MODE_FLAG_NCSYNC |
14774                            DRM_MODE_FLAG_PCSYNC))
14775                 return MODE_HSYNC;
14776
14777         if (mode->flags & (DRM_MODE_FLAG_BCAST |
14778                            DRM_MODE_FLAG_PIXMUX |
14779                            DRM_MODE_FLAG_CLKDIV2))
14780                 return MODE_BAD;
14781
14782         if (INTEL_GEN(dev_priv) >= 9 ||
14783             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
14784                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
14785                 vdisplay_max = 4096;
14786                 htotal_max = 8192;
14787                 vtotal_max = 8192;
14788         } else if (INTEL_GEN(dev_priv) >= 3) {
14789                 hdisplay_max = 4096;
14790                 vdisplay_max = 4096;
14791                 htotal_max = 8192;
14792                 vtotal_max = 8192;
14793         } else {
14794                 hdisplay_max = 2048;
14795                 vdisplay_max = 2048;
14796                 htotal_max = 4096;
14797                 vtotal_max = 4096;
14798         }
14799
14800         if (mode->hdisplay > hdisplay_max ||
14801             mode->hsync_start > htotal_max ||
14802             mode->hsync_end > htotal_max ||
14803             mode->htotal > htotal_max)
14804                 return MODE_H_ILLEGAL;
14805
14806         if (mode->vdisplay > vdisplay_max ||
14807             mode->vsync_start > vtotal_max ||
14808             mode->vsync_end > vtotal_max ||
14809             mode->vtotal > vtotal_max)
14810                 return MODE_V_ILLEGAL;
14811
14812         return MODE_OK;
14813 }
14814
14815 static const struct drm_mode_config_funcs intel_mode_funcs = {
14816         .fb_create = intel_user_framebuffer_create,
14817         .get_format_info = intel_get_format_info,
14818         .output_poll_changed = intel_fbdev_output_poll_changed,
14819         .mode_valid = intel_mode_valid,
14820         .atomic_check = intel_atomic_check,
14821         .atomic_commit = intel_atomic_commit,
14822         .atomic_state_alloc = intel_atomic_state_alloc,
14823         .atomic_state_clear = intel_atomic_state_clear,
14824         .atomic_state_free = intel_atomic_state_free,
14825 };
14826
14827 /**
14828  * intel_init_display_hooks - initialize the display modesetting hooks
14829  * @dev_priv: device private
14830  */
14831 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
14832 {
14833         intel_init_cdclk_hooks(dev_priv);
14834
14835         if (INTEL_GEN(dev_priv) >= 9) {
14836                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14837                 dev_priv->display.get_initial_plane_config =
14838                         skylake_get_initial_plane_config;
14839                 dev_priv->display.crtc_compute_clock =
14840                         haswell_crtc_compute_clock;
14841                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14842                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14843         } else if (HAS_DDI(dev_priv)) {
14844                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14845                 dev_priv->display.get_initial_plane_config =
14846                         i9xx_get_initial_plane_config;
14847                 dev_priv->display.crtc_compute_clock =
14848                         haswell_crtc_compute_clock;
14849                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14850                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14851         } else if (HAS_PCH_SPLIT(dev_priv)) {
14852                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14853                 dev_priv->display.get_initial_plane_config =
14854                         i9xx_get_initial_plane_config;
14855                 dev_priv->display.crtc_compute_clock =
14856                         ironlake_crtc_compute_clock;
14857                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14858                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14859         } else if (IS_CHERRYVIEW(dev_priv)) {
14860                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14861                 dev_priv->display.get_initial_plane_config =
14862                         i9xx_get_initial_plane_config;
14863                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
14864                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14865                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14866         } else if (IS_VALLEYVIEW(dev_priv)) {
14867                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14868                 dev_priv->display.get_initial_plane_config =
14869                         i9xx_get_initial_plane_config;
14870                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
14871                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14872                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14873         } else if (IS_G4X(dev_priv)) {
14874                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14875                 dev_priv->display.get_initial_plane_config =
14876                         i9xx_get_initial_plane_config;
14877                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
14878                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14879                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14880         } else if (IS_PINEVIEW(dev_priv)) {
14881                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14882                 dev_priv->display.get_initial_plane_config =
14883                         i9xx_get_initial_plane_config;
14884                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
14885                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14886                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14887         } else if (!IS_GEN2(dev_priv)) {
14888                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14889                 dev_priv->display.get_initial_plane_config =
14890                         i9xx_get_initial_plane_config;
14891                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14892                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14893                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14894         } else {
14895                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14896                 dev_priv->display.get_initial_plane_config =
14897                         i9xx_get_initial_plane_config;
14898                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
14899                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14900                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14901         }
14902
14903         if (IS_GEN5(dev_priv)) {
14904                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14905         } else if (IS_GEN6(dev_priv)) {
14906                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14907         } else if (IS_IVYBRIDGE(dev_priv)) {
14908                 /* FIXME: detect B0+ stepping and use auto training */
14909                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14910         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
14911                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14912         }
14913
14914         if (INTEL_GEN(dev_priv) >= 9)
14915                 dev_priv->display.update_crtcs = skl_update_crtcs;
14916         else
14917                 dev_priv->display.update_crtcs = intel_update_crtcs;
14918 }
14919
14920 /*
14921  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
14922  */
14923 static void quirk_ssc_force_disable(struct drm_device *dev)
14924 {
14925         struct drm_i915_private *dev_priv = to_i915(dev);
14926         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
14927         DRM_INFO("applying lvds SSC disable quirk\n");
14928 }
14929
14930 /*
14931  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
14932  * brightness value
14933  */
14934 static void quirk_invert_brightness(struct drm_device *dev)
14935 {
14936         struct drm_i915_private *dev_priv = to_i915(dev);
14937         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
14938         DRM_INFO("applying inverted panel brightness quirk\n");
14939 }
14940
14941 /* Some VBT's incorrectly indicate no backlight is present */
14942 static void quirk_backlight_present(struct drm_device *dev)
14943 {
14944         struct drm_i915_private *dev_priv = to_i915(dev);
14945         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
14946         DRM_INFO("applying backlight present quirk\n");
14947 }
14948
14949 /* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms
14950  * which is 300 ms greater than eDP spec T12 min.
14951  */
14952 static void quirk_increase_t12_delay(struct drm_device *dev)
14953 {
14954         struct drm_i915_private *dev_priv = to_i915(dev);
14955
14956         dev_priv->quirks |= QUIRK_INCREASE_T12_DELAY;
14957         DRM_INFO("Applying T12 delay quirk\n");
14958 }
14959
14960 /*
14961  * GeminiLake NUC HDMI outputs require additional off time
14962  * this allows the onboard retimer to correctly sync to signal
14963  */
14964 static void quirk_increase_ddi_disabled_time(struct drm_device *dev)
14965 {
14966         struct drm_i915_private *dev_priv = to_i915(dev);
14967
14968         dev_priv->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME;
14969         DRM_INFO("Applying Increase DDI Disabled quirk\n");
14970 }
14971
14972 struct intel_quirk {
14973         int device;
14974         int subsystem_vendor;
14975         int subsystem_device;
14976         void (*hook)(struct drm_device *dev);
14977 };
14978
14979 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
14980 struct intel_dmi_quirk {
14981         void (*hook)(struct drm_device *dev);
14982         const struct dmi_system_id (*dmi_id_list)[];
14983 };
14984
14985 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
14986 {
14987         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
14988         return 1;
14989 }
14990
14991 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
14992         {
14993                 .dmi_id_list = &(const struct dmi_system_id[]) {
14994                         {
14995                                 .callback = intel_dmi_reverse_brightness,
14996                                 .ident = "NCR Corporation",
14997                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
14998                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
14999                                 },
15000                         },
15001                         { }  /* terminating entry */
15002                 },
15003                 .hook = quirk_invert_brightness,
15004         },
15005 };
15006
15007 static struct intel_quirk intel_quirks[] = {
15008         /* Lenovo U160 cannot use SSC on LVDS */
15009         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
15010
15011         /* Sony Vaio Y cannot use SSC on LVDS */
15012         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
15013
15014         /* Acer Aspire 5734Z must invert backlight brightness */
15015         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
15016
15017         /* Acer/eMachines G725 */
15018         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
15019
15020         /* Acer/eMachines e725 */
15021         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
15022
15023         /* Acer/Packard Bell NCL20 */
15024         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
15025
15026         /* Acer Aspire 4736Z */
15027         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
15028
15029         /* Acer Aspire 5336 */
15030         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
15031
15032         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
15033         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
15034
15035         /* Acer C720 Chromebook (Core i3 4005U) */
15036         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
15037
15038         /* Apple Macbook 2,1 (Core 2 T7400) */
15039         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
15040
15041         /* Apple Macbook 4,1 */
15042         { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present },
15043
15044         /* Toshiba CB35 Chromebook (Celeron 2955U) */
15045         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
15046
15047         /* HP Chromebook 14 (Celeron 2955U) */
15048         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
15049
15050         /* Dell Chromebook 11 */
15051         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
15052
15053         /* Dell Chromebook 11 (2015 version) */
15054         { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
15055
15056         /* Toshiba Satellite P50-C-18C */
15057         { 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay },
15058
15059         /* GeminiLake NUC */
15060         { 0x3185, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
15061         { 0x3184, 0x8086, 0x2072, quirk_increase_ddi_disabled_time },
15062         /* ASRock ITX*/
15063         { 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
15064         { 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time },
15065 };
15066
15067 static void intel_init_quirks(struct drm_device *dev)
15068 {
15069         struct pci_dev *d = dev->pdev;
15070         int i;
15071
15072         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
15073                 struct intel_quirk *q = &intel_quirks[i];
15074
15075                 if (d->device == q->device &&
15076                     (d->subsystem_vendor == q->subsystem_vendor ||
15077                      q->subsystem_vendor == PCI_ANY_ID) &&
15078                     (d->subsystem_device == q->subsystem_device ||
15079                      q->subsystem_device == PCI_ANY_ID))
15080                         q->hook(dev);
15081         }
15082         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
15083                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
15084                         intel_dmi_quirks[i].hook(dev);
15085         }
15086 }
15087
15088 /* Disable the VGA plane that we never use */
15089 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15090 {
15091         struct pci_dev *pdev = dev_priv->drm.pdev;
15092         u8 sr1;
15093         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15094
15095         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15096         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15097         outb(SR01, VGA_SR_INDEX);
15098         sr1 = inb(VGA_SR_DATA);
15099         outb(sr1 | 1<<5, VGA_SR_DATA);
15100         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15101         udelay(300);
15102
15103         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15104         POSTING_READ(vga_reg);
15105 }
15106
15107 void intel_modeset_init_hw(struct drm_device *dev)
15108 {
15109         struct drm_i915_private *dev_priv = to_i915(dev);
15110
15111         intel_update_cdclk(dev_priv);
15112         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15113         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15114 }
15115
15116 /*
15117  * Calculate what we think the watermarks should be for the state we've read
15118  * out of the hardware and then immediately program those watermarks so that
15119  * we ensure the hardware settings match our internal state.
15120  *
15121  * We can calculate what we think WM's should be by creating a duplicate of the
15122  * current state (which was constructed during hardware readout) and running it
15123  * through the atomic check code to calculate new watermark values in the
15124  * state object.
15125  */
15126 static void sanitize_watermarks(struct drm_device *dev)
15127 {
15128         struct drm_i915_private *dev_priv = to_i915(dev);
15129         struct drm_atomic_state *state;
15130         struct intel_atomic_state *intel_state;
15131         struct drm_crtc *crtc;
15132         struct drm_crtc_state *cstate;
15133         struct drm_modeset_acquire_ctx ctx;
15134         int ret;
15135         int i;
15136
15137         /* Only supported on platforms that use atomic watermark design */
15138         if (!dev_priv->display.optimize_watermarks)
15139                 return;
15140
15141         /*
15142          * We need to hold connection_mutex before calling duplicate_state so
15143          * that the connector loop is protected.
15144          */
15145         drm_modeset_acquire_init(&ctx, 0);
15146 retry:
15147         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15148         if (ret == -EDEADLK) {
15149                 drm_modeset_backoff(&ctx);
15150                 goto retry;
15151         } else if (WARN_ON(ret)) {
15152                 goto fail;
15153         }
15154
15155         state = drm_atomic_helper_duplicate_state(dev, &ctx);
15156         if (WARN_ON(IS_ERR(state)))
15157                 goto fail;
15158
15159         intel_state = to_intel_atomic_state(state);
15160
15161         /*
15162          * Hardware readout is the only time we don't want to calculate
15163          * intermediate watermarks (since we don't trust the current
15164          * watermarks).
15165          */
15166         if (!HAS_GMCH_DISPLAY(dev_priv))
15167                 intel_state->skip_intermediate_wm = true;
15168
15169         ret = intel_atomic_check(dev, state);
15170         if (ret) {
15171                 /*
15172                  * If we fail here, it means that the hardware appears to be
15173                  * programmed in a way that shouldn't be possible, given our
15174                  * understanding of watermark requirements.  This might mean a
15175                  * mistake in the hardware readout code or a mistake in the
15176                  * watermark calculations for a given platform.  Raise a WARN
15177                  * so that this is noticeable.
15178                  *
15179                  * If this actually happens, we'll have to just leave the
15180                  * BIOS-programmed watermarks untouched and hope for the best.
15181                  */
15182                 WARN(true, "Could not determine valid watermarks for inherited state\n");
15183                 goto put_state;
15184         }
15185
15186         /* Write calculated watermark values back */
15187         for_each_new_crtc_in_state(state, crtc, cstate, i) {
15188                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
15189
15190                 cs->wm.need_postvbl_update = true;
15191                 dev_priv->display.optimize_watermarks(intel_state, cs);
15192
15193                 to_intel_crtc_state(crtc->state)->wm = cs->wm;
15194         }
15195
15196 put_state:
15197         drm_atomic_state_put(state);
15198 fail:
15199         drm_modeset_drop_locks(&ctx);
15200         drm_modeset_acquire_fini(&ctx);
15201 }
15202
15203 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
15204 {
15205         if (IS_GEN5(dev_priv)) {
15206                 u32 fdi_pll_clk =
15207                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
15208
15209                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
15210         } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
15211                 dev_priv->fdi_pll_freq = 270000;
15212         } else {
15213                 return;
15214         }
15215
15216         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
15217 }
15218
15219 static int intel_initial_commit(struct drm_device *dev)
15220 {
15221         struct drm_atomic_state *state = NULL;
15222         struct drm_modeset_acquire_ctx ctx;
15223         struct drm_crtc *crtc;
15224         struct drm_crtc_state *crtc_state;
15225         int ret = 0;
15226
15227         state = drm_atomic_state_alloc(dev);
15228         if (!state)
15229                 return -ENOMEM;
15230
15231         drm_modeset_acquire_init(&ctx, 0);
15232
15233 retry:
15234         state->acquire_ctx = &ctx;
15235
15236         drm_for_each_crtc(crtc, dev) {
15237                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
15238                 if (IS_ERR(crtc_state)) {
15239                         ret = PTR_ERR(crtc_state);
15240                         goto out;
15241                 }
15242
15243                 if (crtc_state->active) {
15244                         ret = drm_atomic_add_affected_planes(state, crtc);
15245                         if (ret)
15246                                 goto out;
15247                 }
15248         }
15249
15250         ret = drm_atomic_commit(state);
15251
15252 out:
15253         if (ret == -EDEADLK) {
15254                 drm_atomic_state_clear(state);
15255                 drm_modeset_backoff(&ctx);
15256                 goto retry;
15257         }
15258
15259         drm_atomic_state_put(state);
15260
15261         drm_modeset_drop_locks(&ctx);
15262         drm_modeset_acquire_fini(&ctx);
15263
15264         return ret;
15265 }
15266
15267 int intel_modeset_init(struct drm_device *dev)
15268 {
15269         struct drm_i915_private *dev_priv = to_i915(dev);
15270         struct i915_ggtt *ggtt = &dev_priv->ggtt;
15271         enum pipe pipe;
15272         struct intel_crtc *crtc;
15273         int ret;
15274
15275         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
15276
15277         drm_mode_config_init(dev);
15278
15279         dev->mode_config.min_width = 0;
15280         dev->mode_config.min_height = 0;
15281
15282         dev->mode_config.preferred_depth = 24;
15283         dev->mode_config.prefer_shadow = 1;
15284
15285         dev->mode_config.allow_fb_modifiers = true;
15286
15287         dev->mode_config.funcs = &intel_mode_funcs;
15288
15289         init_llist_head(&dev_priv->atomic_helper.free_list);
15290         INIT_WORK(&dev_priv->atomic_helper.free_work,
15291                   intel_atomic_helper_free_state_worker);
15292
15293         intel_init_quirks(dev);
15294
15295         intel_init_pm(dev_priv);
15296
15297         /*
15298          * There may be no VBT; and if the BIOS enabled SSC we can
15299          * just keep using it to avoid unnecessary flicker.  Whereas if the
15300          * BIOS isn't using it, don't assume it will work even if the VBT
15301          * indicates as much.
15302          */
15303         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
15304                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15305                                             DREF_SSC1_ENABLE);
15306
15307                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
15308                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
15309                                      bios_lvds_use_ssc ? "en" : "dis",
15310                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
15311                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
15312                 }
15313         }
15314
15315         /* maximum framebuffer dimensions */
15316         if (IS_GEN2(dev_priv)) {
15317                 dev->mode_config.max_width = 2048;
15318                 dev->mode_config.max_height = 2048;
15319         } else if (IS_GEN3(dev_priv)) {
15320                 dev->mode_config.max_width = 4096;
15321                 dev->mode_config.max_height = 4096;
15322         } else {
15323                 dev->mode_config.max_width = 8192;
15324                 dev->mode_config.max_height = 8192;
15325         }
15326
15327         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15328                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
15329                 dev->mode_config.cursor_height = 1023;
15330         } else if (IS_GEN2(dev_priv)) {
15331                 dev->mode_config.cursor_width = 64;
15332                 dev->mode_config.cursor_height = 64;
15333         } else {
15334                 dev->mode_config.cursor_width = 256;
15335                 dev->mode_config.cursor_height = 256;
15336         }
15337
15338         dev->mode_config.fb_base = ggtt->gmadr.start;
15339
15340         DRM_DEBUG_KMS("%d display pipe%s available.\n",
15341                       INTEL_INFO(dev_priv)->num_pipes,
15342                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
15343
15344         for_each_pipe(dev_priv, pipe) {
15345                 ret = intel_crtc_init(dev_priv, pipe);
15346                 if (ret) {
15347                         drm_mode_config_cleanup(dev);
15348                         return ret;
15349                 }
15350         }
15351
15352         intel_shared_dpll_init(dev);
15353         intel_update_fdi_pll_freq(dev_priv);
15354
15355         intel_update_czclk(dev_priv);
15356         intel_modeset_init_hw(dev);
15357
15358         if (dev_priv->max_cdclk_freq == 0)
15359                 intel_update_max_cdclk(dev_priv);
15360
15361         /* Just disable it once at startup */
15362         i915_disable_vga(dev_priv);
15363         intel_setup_outputs(dev_priv);
15364
15365         drm_modeset_lock_all(dev);
15366         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
15367         drm_modeset_unlock_all(dev);
15368
15369         for_each_intel_crtc(dev, crtc) {
15370                 struct intel_initial_plane_config plane_config = {};
15371
15372                 if (!crtc->active)
15373                         continue;
15374
15375                 /*
15376                  * Note that reserving the BIOS fb up front prevents us
15377                  * from stuffing other stolen allocations like the ring
15378                  * on top.  This prevents some ugliness at boot time, and
15379                  * can even allow for smooth boot transitions if the BIOS
15380                  * fb is large enough for the active pipe configuration.
15381                  */
15382                 dev_priv->display.get_initial_plane_config(crtc,
15383                                                            &plane_config);
15384
15385                 /*
15386                  * If the fb is shared between multiple heads, we'll
15387                  * just get the first one.
15388                  */
15389                 intel_find_initial_plane_obj(crtc, &plane_config);
15390         }
15391
15392         /*
15393          * Make sure hardware watermarks really match the state we read out.
15394          * Note that we need to do this after reconstructing the BIOS fb's
15395          * since the watermark calculation done here will use pstate->fb.
15396          */
15397         if (!HAS_GMCH_DISPLAY(dev_priv))
15398                 sanitize_watermarks(dev);
15399
15400         /*
15401          * Force all active planes to recompute their states. So that on
15402          * mode_setcrtc after probe, all the intel_plane_state variables
15403          * are already calculated and there is no assert_plane warnings
15404          * during bootup.
15405          */
15406         ret = intel_initial_commit(dev);
15407         if (ret)
15408                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
15409
15410         return 0;
15411 }
15412
15413 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15414 {
15415         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15416         /* 640x480@60Hz, ~25175 kHz */
15417         struct dpll clock = {
15418                 .m1 = 18,
15419                 .m2 = 7,
15420                 .p1 = 13,
15421                 .p2 = 4,
15422                 .n = 2,
15423         };
15424         u32 dpll, fp;
15425         int i;
15426
15427         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
15428
15429         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
15430                       pipe_name(pipe), clock.vco, clock.dot);
15431
15432         fp = i9xx_dpll_compute_fp(&clock);
15433         dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
15434                 DPLL_VGA_MODE_DIS |
15435                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
15436                 PLL_P2_DIVIDE_BY_4 |
15437                 PLL_REF_INPUT_DREFCLK |
15438                 DPLL_VCO_ENABLE;
15439
15440         I915_WRITE(FP0(pipe), fp);
15441         I915_WRITE(FP1(pipe), fp);
15442
15443         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
15444         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
15445         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
15446         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
15447         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
15448         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
15449         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
15450
15451         /*
15452          * Apparently we need to have VGA mode enabled prior to changing
15453          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
15454          * dividers, even though the register value does change.
15455          */
15456         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
15457         I915_WRITE(DPLL(pipe), dpll);
15458
15459         /* Wait for the clocks to stabilize. */
15460         POSTING_READ(DPLL(pipe));
15461         udelay(150);
15462
15463         /* The pixel multiplier can only be updated once the
15464          * DPLL is enabled and the clocks are stable.
15465          *
15466          * So write it again.
15467          */
15468         I915_WRITE(DPLL(pipe), dpll);
15469
15470         /* We do this three times for luck */
15471         for (i = 0; i < 3 ; i++) {
15472                 I915_WRITE(DPLL(pipe), dpll);
15473                 POSTING_READ(DPLL(pipe));
15474                 udelay(150); /* wait for warmup */
15475         }
15476
15477         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
15478         POSTING_READ(PIPECONF(pipe));
15479
15480         intel_wait_for_pipe_scanline_moving(crtc);
15481 }
15482
15483 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15484 {
15485         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15486
15487         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
15488                       pipe_name(pipe));
15489
15490         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
15491         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
15492         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
15493         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
15494         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
15495
15496         I915_WRITE(PIPECONF(pipe), 0);
15497         POSTING_READ(PIPECONF(pipe));
15498
15499         intel_wait_for_pipe_scanline_stopped(crtc);
15500
15501         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
15502         POSTING_READ(DPLL(pipe));
15503 }
15504
15505 static void
15506 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
15507 {
15508         struct intel_crtc *crtc;
15509
15510         if (INTEL_GEN(dev_priv) >= 4)
15511                 return;
15512
15513         for_each_intel_crtc(&dev_priv->drm, crtc) {
15514                 struct intel_plane *plane =
15515                         to_intel_plane(crtc->base.primary);
15516                 struct intel_crtc *plane_crtc;
15517                 enum pipe pipe;
15518
15519                 if (!plane->get_hw_state(plane, &pipe))
15520                         continue;
15521
15522                 if (pipe == crtc->pipe)
15523                         continue;
15524
15525                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
15526                               plane->base.base.id, plane->base.name);
15527
15528                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15529                 intel_plane_disable_noatomic(plane_crtc, plane);
15530         }
15531 }
15532
15533 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15534 {
15535         struct drm_device *dev = crtc->base.dev;
15536         struct intel_encoder *encoder;
15537
15538         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15539                 return true;
15540
15541         return false;
15542 }
15543
15544 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
15545 {
15546         struct drm_device *dev = encoder->base.dev;
15547         struct intel_connector *connector;
15548
15549         for_each_connector_on_encoder(dev, &encoder->base, connector)
15550                 return connector;
15551
15552         return NULL;
15553 }
15554
15555 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
15556                               enum pipe pch_transcoder)
15557 {
15558         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
15559                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
15560 }
15561
15562 static void intel_sanitize_crtc(struct intel_crtc *crtc,
15563                                 struct drm_modeset_acquire_ctx *ctx)
15564 {
15565         struct drm_device *dev = crtc->base.dev;
15566         struct drm_i915_private *dev_priv = to_i915(dev);
15567         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
15568
15569         /* Clear any frame start delays used for debugging left by the BIOS */
15570         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
15571                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15572
15573                 I915_WRITE(reg,
15574                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15575         }
15576
15577         if (crtc->active) {
15578                 struct intel_plane *plane;
15579
15580                 /* Disable everything but the primary plane */
15581                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15582                         const struct intel_plane_state *plane_state =
15583                                 to_intel_plane_state(plane->base.state);
15584
15585                         if (plane_state->base.visible &&
15586                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
15587                                 intel_plane_disable_noatomic(crtc, plane);
15588                 }
15589         }
15590
15591         /* Adjust the state of the output pipe according to whether we
15592          * have active connectors/encoders. */
15593         if (crtc->active && !intel_crtc_has_encoders(crtc))
15594                 intel_crtc_disable_noatomic(&crtc->base, ctx);
15595
15596         if (crtc->active || HAS_GMCH_DISPLAY(dev_priv)) {
15597                 /*
15598                  * We start out with underrun reporting disabled to avoid races.
15599                  * For correct bookkeeping mark this on active crtcs.
15600                  *
15601                  * Also on gmch platforms we dont have any hardware bits to
15602                  * disable the underrun reporting. Which means we need to start
15603                  * out with underrun reporting disabled also on inactive pipes,
15604                  * since otherwise we'll complain about the garbage we read when
15605                  * e.g. coming up after runtime pm.
15606                  *
15607                  * No protection against concurrent access is required - at
15608                  * worst a fifo underrun happens which also sets this to false.
15609                  */
15610                 crtc->cpu_fifo_underrun_disabled = true;
15611                 /*
15612                  * We track the PCH trancoder underrun reporting state
15613                  * within the crtc. With crtc for pipe A housing the underrun
15614                  * reporting state for PCH transcoder A, crtc for pipe B housing
15615                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
15616                  * and marking underrun reporting as disabled for the non-existing
15617                  * PCH transcoders B and C would prevent enabling the south
15618                  * error interrupt (see cpt_can_enable_serr_int()).
15619                  */
15620                 if (has_pch_trancoder(dev_priv, crtc->pipe))
15621                         crtc->pch_fifo_underrun_disabled = true;
15622         }
15623 }
15624
15625 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15626 {
15627         struct intel_connector *connector;
15628
15629         /* We need to check both for a crtc link (meaning that the
15630          * encoder is active and trying to read from a pipe) and the
15631          * pipe itself being active. */
15632         bool has_active_crtc = encoder->base.crtc &&
15633                 to_intel_crtc(encoder->base.crtc)->active;
15634
15635         connector = intel_encoder_find_connector(encoder);
15636         if (connector && !has_active_crtc) {
15637                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15638                               encoder->base.base.id,
15639                               encoder->base.name);
15640
15641                 /* Connector is active, but has no active pipe. This is
15642                  * fallout from our resume register restoring. Disable
15643                  * the encoder manually again. */
15644                 if (encoder->base.crtc) {
15645                         struct drm_crtc_state *crtc_state = encoder->base.crtc->state;
15646
15647                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15648                                       encoder->base.base.id,
15649                                       encoder->base.name);
15650                         encoder->disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15651                         if (encoder->post_disable)
15652                                 encoder->post_disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15653                 }
15654                 encoder->base.crtc = NULL;
15655
15656                 /* Inconsistent output/port/pipe state happens presumably due to
15657                  * a bug in one of the get_hw_state functions. Or someplace else
15658                  * in our code, like the register restore mess on resume. Clamp
15659                  * things to off as a safer default. */
15660
15661                 connector->base.dpms = DRM_MODE_DPMS_OFF;
15662                 connector->base.encoder = NULL;
15663         }
15664
15665         /* notify opregion of the sanitized encoder state */
15666         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
15667 }
15668
15669 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
15670 {
15671         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15672
15673         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15674                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15675                 i915_disable_vga(dev_priv);
15676         }
15677 }
15678
15679 void i915_redisable_vga(struct drm_i915_private *dev_priv)
15680 {
15681         /* This function can be called both from intel_modeset_setup_hw_state or
15682          * at a very early point in our resume sequence, where the power well
15683          * structures are not yet restored. Since this function is at a very
15684          * paranoid "someone might have enabled VGA while we were not looking"
15685          * level, just check if the power well is enabled instead of trying to
15686          * follow the "don't touch the power well if we don't need it" policy
15687          * the rest of the driver uses. */
15688         if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
15689                 return;
15690
15691         i915_redisable_vga_power_on(dev_priv);
15692
15693         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
15694 }
15695
15696 /* FIXME read out full plane state for all planes */
15697 static void readout_plane_state(struct drm_i915_private *dev_priv)
15698 {
15699         struct intel_plane *plane;
15700         struct intel_crtc *crtc;
15701
15702         for_each_intel_plane(&dev_priv->drm, plane) {
15703                 struct intel_plane_state *plane_state =
15704                         to_intel_plane_state(plane->base.state);
15705                 struct intel_crtc_state *crtc_state;
15706                 enum pipe pipe = PIPE_A;
15707                 bool visible;
15708
15709                 visible = plane->get_hw_state(plane, &pipe);
15710
15711                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15712                 crtc_state = to_intel_crtc_state(crtc->base.state);
15713
15714                 intel_set_plane_visible(crtc_state, plane_state, visible);
15715
15716                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
15717                               plane->base.base.id, plane->base.name,
15718                               enableddisabled(visible), pipe_name(pipe));
15719         }
15720
15721         for_each_intel_crtc(&dev_priv->drm, crtc) {
15722                 struct intel_crtc_state *crtc_state =
15723                         to_intel_crtc_state(crtc->base.state);
15724
15725                 fixup_active_planes(crtc_state);
15726         }
15727 }
15728
15729 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15730 {
15731         struct drm_i915_private *dev_priv = to_i915(dev);
15732         enum pipe pipe;
15733         struct intel_crtc *crtc;
15734         struct intel_encoder *encoder;
15735         struct intel_connector *connector;
15736         struct drm_connector_list_iter conn_iter;
15737         int i;
15738
15739         dev_priv->active_crtcs = 0;
15740
15741         for_each_intel_crtc(dev, crtc) {
15742                 struct intel_crtc_state *crtc_state =
15743                         to_intel_crtc_state(crtc->base.state);
15744
15745                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
15746                 memset(crtc_state, 0, sizeof(*crtc_state));
15747                 crtc_state->base.crtc = &crtc->base;
15748
15749                 crtc_state->base.active = crtc_state->base.enable =
15750                         dev_priv->display.get_pipe_config(crtc, crtc_state);
15751
15752                 crtc->base.enabled = crtc_state->base.enable;
15753                 crtc->active = crtc_state->base.active;
15754
15755                 if (crtc_state->base.active)
15756                         dev_priv->active_crtcs |= 1 << crtc->pipe;
15757
15758                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
15759                               crtc->base.base.id, crtc->base.name,
15760                               enableddisabled(crtc_state->base.active));
15761         }
15762
15763         readout_plane_state(dev_priv);
15764
15765         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15766                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15767
15768                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
15769                                                         &pll->state.hw_state);
15770                 pll->state.crtc_mask = 0;
15771                 for_each_intel_crtc(dev, crtc) {
15772                         struct intel_crtc_state *crtc_state =
15773                                 to_intel_crtc_state(crtc->base.state);
15774
15775                         if (crtc_state->base.active &&
15776                             crtc_state->shared_dpll == pll)
15777                                 pll->state.crtc_mask |= 1 << crtc->pipe;
15778                 }
15779                 pll->active_mask = pll->state.crtc_mask;
15780
15781                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15782                               pll->info->name, pll->state.crtc_mask, pll->on);
15783         }
15784
15785         for_each_intel_encoder(dev, encoder) {
15786                 pipe = 0;
15787
15788                 if (encoder->get_hw_state(encoder, &pipe)) {
15789                         struct intel_crtc_state *crtc_state;
15790
15791                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15792                         crtc_state = to_intel_crtc_state(crtc->base.state);
15793
15794                         encoder->base.crtc = &crtc->base;
15795                         encoder->get_config(encoder, crtc_state);
15796                 } else {
15797                         encoder->base.crtc = NULL;
15798                 }
15799
15800                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15801                               encoder->base.base.id, encoder->base.name,
15802                               enableddisabled(encoder->base.crtc),
15803                               pipe_name(pipe));
15804         }
15805
15806         drm_connector_list_iter_begin(dev, &conn_iter);
15807         for_each_intel_connector_iter(connector, &conn_iter) {
15808                 if (connector->get_hw_state(connector)) {
15809                         connector->base.dpms = DRM_MODE_DPMS_ON;
15810
15811                         encoder = connector->encoder;
15812                         connector->base.encoder = &encoder->base;
15813
15814                         if (encoder->base.crtc &&
15815                             encoder->base.crtc->state->active) {
15816                                 /*
15817                                  * This has to be done during hardware readout
15818                                  * because anything calling .crtc_disable may
15819                                  * rely on the connector_mask being accurate.
15820                                  */
15821                                 encoder->base.crtc->state->connector_mask |=
15822                                         drm_connector_mask(&connector->base);
15823                                 encoder->base.crtc->state->encoder_mask |=
15824                                         drm_encoder_mask(&encoder->base);
15825                         }
15826
15827                 } else {
15828                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15829                         connector->base.encoder = NULL;
15830                 }
15831                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15832                               connector->base.base.id, connector->base.name,
15833                               enableddisabled(connector->base.encoder));
15834         }
15835         drm_connector_list_iter_end(&conn_iter);
15836
15837         for_each_intel_crtc(dev, crtc) {
15838                 struct intel_crtc_state *crtc_state =
15839                         to_intel_crtc_state(crtc->base.state);
15840                 int min_cdclk = 0;
15841
15842                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15843                 if (crtc_state->base.active) {
15844                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
15845                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
15846                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
15847                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
15848                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15849
15850                         /*
15851                          * The initial mode needs to be set in order to keep
15852                          * the atomic core happy. It wants a valid mode if the
15853                          * crtc's enabled, so we do the above call.
15854                          *
15855                          * But we don't set all the derived state fully, hence
15856                          * set a flag to indicate that a full recalculation is
15857                          * needed on the next commit.
15858                          */
15859                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
15860
15861                         intel_crtc_compute_pixel_rate(crtc_state);
15862
15863                         if (dev_priv->display.modeset_calc_cdclk) {
15864                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
15865                                 if (WARN_ON(min_cdclk < 0))
15866                                         min_cdclk = 0;
15867                         }
15868
15869                         drm_calc_timestamping_constants(&crtc->base,
15870                                                         &crtc_state->base.adjusted_mode);
15871                         update_scanline_offset(crtc);
15872                 }
15873
15874                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
15875                 dev_priv->min_voltage_level[crtc->pipe] =
15876                         crtc_state->min_voltage_level;
15877
15878                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
15879         }
15880 }
15881
15882 static void
15883 get_encoder_power_domains(struct drm_i915_private *dev_priv)
15884 {
15885         struct intel_encoder *encoder;
15886
15887         for_each_intel_encoder(&dev_priv->drm, encoder) {
15888                 u64 get_domains;
15889                 enum intel_display_power_domain domain;
15890                 struct intel_crtc_state *crtc_state;
15891
15892                 if (!encoder->get_power_domains)
15893                         continue;
15894
15895                 /*
15896                  * MST-primary and inactive encoders don't have a crtc state
15897                  * and neither of these require any power domain references.
15898                  */
15899                 if (!encoder->base.crtc)
15900                         continue;
15901
15902                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
15903                 get_domains = encoder->get_power_domains(encoder, crtc_state);
15904                 for_each_power_domain(domain, get_domains)
15905                         intel_display_power_get(dev_priv, domain);
15906         }
15907 }
15908
15909 static void intel_early_display_was(struct drm_i915_private *dev_priv)
15910 {
15911         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
15912         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
15913                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
15914                            DARBF_GATING_DIS);
15915
15916         if (IS_HASWELL(dev_priv)) {
15917                 /*
15918                  * WaRsPkgCStateDisplayPMReq:hsw
15919                  * System hang if this isn't done before disabling all planes!
15920                  */
15921                 I915_WRITE(CHICKEN_PAR1_1,
15922                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
15923         }
15924 }
15925
15926 /* Scan out the current hw modeset state,
15927  * and sanitizes it to the current state
15928  */
15929 static void
15930 intel_modeset_setup_hw_state(struct drm_device *dev,
15931                              struct drm_modeset_acquire_ctx *ctx)
15932 {
15933         struct drm_i915_private *dev_priv = to_i915(dev);
15934         struct intel_crtc *crtc;
15935         struct intel_encoder *encoder;
15936         int i;
15937
15938         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
15939
15940         intel_early_display_was(dev_priv);
15941         intel_modeset_readout_hw_state(dev);
15942
15943         /* HW state is read out, now we need to sanitize this mess. */
15944         get_encoder_power_domains(dev_priv);
15945
15946         /*
15947          * intel_sanitize_plane_mapping() may need to do vblank
15948          * waits, so we need vblank interrupts restored beforehand.
15949          */
15950         for_each_intel_crtc(&dev_priv->drm, crtc) {
15951                 drm_crtc_vblank_reset(&crtc->base);
15952
15953                 if (crtc->active)
15954                         drm_crtc_vblank_on(&crtc->base);
15955         }
15956
15957         intel_sanitize_plane_mapping(dev_priv);
15958
15959         for_each_intel_encoder(dev, encoder)
15960                 intel_sanitize_encoder(encoder);
15961
15962         for_each_intel_crtc(&dev_priv->drm, crtc) {
15963                 intel_sanitize_crtc(crtc, ctx);
15964                 intel_dump_pipe_config(crtc, crtc->config,
15965                                        "[setup_hw_state]");
15966         }
15967
15968         intel_modeset_update_connector_atomic_state(dev);
15969
15970         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15971                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15972
15973                 if (!pll->on || pll->active_mask)
15974                         continue;
15975
15976                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
15977                               pll->info->name);
15978
15979                 pll->info->funcs->disable(dev_priv, pll);
15980                 pll->on = false;
15981         }
15982
15983         if (IS_G4X(dev_priv)) {
15984                 g4x_wm_get_hw_state(dev);
15985                 g4x_wm_sanitize(dev_priv);
15986         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15987                 vlv_wm_get_hw_state(dev);
15988                 vlv_wm_sanitize(dev_priv);
15989         } else if (INTEL_GEN(dev_priv) >= 9) {
15990                 skl_wm_get_hw_state(dev);
15991         } else if (HAS_PCH_SPLIT(dev_priv)) {
15992                 ilk_wm_get_hw_state(dev);
15993         }
15994
15995         for_each_intel_crtc(dev, crtc) {
15996                 u64 put_domains;
15997
15998                 put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc->config);
15999                 if (WARN_ON(put_domains))
16000                         modeset_put_power_domains(dev_priv, put_domains);
16001         }
16002
16003         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
16004
16005         intel_fbc_init_pipe_state(dev_priv);
16006 }
16007
16008 void intel_display_resume(struct drm_device *dev)
16009 {
16010         struct drm_i915_private *dev_priv = to_i915(dev);
16011         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
16012         struct drm_modeset_acquire_ctx ctx;
16013         int ret;
16014
16015         dev_priv->modeset_restore_state = NULL;
16016         if (state)
16017                 state->acquire_ctx = &ctx;
16018
16019         drm_modeset_acquire_init(&ctx, 0);
16020
16021         while (1) {
16022                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
16023                 if (ret != -EDEADLK)
16024                         break;
16025
16026                 drm_modeset_backoff(&ctx);
16027         }
16028
16029         if (!ret)
16030                 ret = __intel_display_resume(dev, state, &ctx);
16031
16032         intel_enable_ipc(dev_priv);
16033         drm_modeset_drop_locks(&ctx);
16034         drm_modeset_acquire_fini(&ctx);
16035
16036         if (ret)
16037                 DRM_ERROR("Restoring old state failed with %i\n", ret);
16038         if (state)
16039                 drm_atomic_state_put(state);
16040 }
16041
16042 int intel_connector_register(struct drm_connector *connector)
16043 {
16044         struct intel_connector *intel_connector = to_intel_connector(connector);
16045         int ret;
16046
16047         ret = intel_backlight_device_register(intel_connector);
16048         if (ret)
16049                 goto err;
16050
16051         return 0;
16052
16053 err:
16054         return ret;
16055 }
16056
16057 void intel_connector_unregister(struct drm_connector *connector)
16058 {
16059         struct intel_connector *intel_connector = to_intel_connector(connector);
16060
16061         intel_backlight_device_unregister(intel_connector);
16062         intel_panel_destroy_backlight(connector);
16063 }
16064
16065 static void intel_hpd_poll_fini(struct drm_device *dev)
16066 {
16067         struct intel_connector *connector;
16068         struct drm_connector_list_iter conn_iter;
16069
16070         /* Kill all the work that may have been queued by hpd. */
16071         drm_connector_list_iter_begin(dev, &conn_iter);
16072         for_each_intel_connector_iter(connector, &conn_iter) {
16073                 if (connector->modeset_retry_work.func)
16074                         cancel_work_sync(&connector->modeset_retry_work);
16075                 if (connector->hdcp_shim) {
16076                         cancel_delayed_work_sync(&connector->hdcp_check_work);
16077                         cancel_work_sync(&connector->hdcp_prop_work);
16078                 }
16079         }
16080         drm_connector_list_iter_end(&conn_iter);
16081 }
16082
16083 void intel_modeset_cleanup(struct drm_device *dev)
16084 {
16085         struct drm_i915_private *dev_priv = to_i915(dev);
16086
16087         flush_workqueue(dev_priv->modeset_wq);
16088
16089         flush_work(&dev_priv->atomic_helper.free_work);
16090         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
16091
16092         /*
16093          * Interrupts and polling as the first thing to avoid creating havoc.
16094          * Too much stuff here (turning of connectors, ...) would
16095          * experience fancy races otherwise.
16096          */
16097         intel_irq_uninstall(dev_priv);
16098
16099         /*
16100          * Due to the hpd irq storm handling the hotplug work can re-arm the
16101          * poll handlers. Hence disable polling after hpd handling is shut down.
16102          */
16103         intel_hpd_poll_fini(dev);
16104
16105         /* poll work can call into fbdev, hence clean that up afterwards */
16106         intel_fbdev_fini(dev_priv);
16107
16108         intel_unregister_dsm_handler();
16109
16110         intel_fbc_global_disable(dev_priv);
16111
16112         /* flush any delayed tasks or pending work */
16113         flush_scheduled_work();
16114
16115         drm_mode_config_cleanup(dev);
16116
16117         intel_cleanup_overlay(dev_priv);
16118
16119         intel_teardown_gmbus(dev_priv);
16120
16121         destroy_workqueue(dev_priv->modeset_wq);
16122 }
16123
16124 void intel_connector_attach_encoder(struct intel_connector *connector,
16125                                     struct intel_encoder *encoder)
16126 {
16127         connector->encoder = encoder;
16128         drm_connector_attach_encoder(&connector->base, &encoder->base);
16129 }
16130
16131 /*
16132  * set vga decode state - true == enable VGA decode
16133  */
16134 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
16135 {
16136         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
16137         u16 gmch_ctrl;
16138
16139         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
16140                 DRM_ERROR("failed to read control word\n");
16141                 return -EIO;
16142         }
16143
16144         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
16145                 return 0;
16146
16147         if (state)
16148                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
16149         else
16150                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
16151
16152         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
16153                 DRM_ERROR("failed to write control word\n");
16154                 return -EIO;
16155         }
16156
16157         return 0;
16158 }
16159
16160 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
16161
16162 struct intel_display_error_state {
16163
16164         u32 power_well_driver;
16165
16166         int num_transcoders;
16167
16168         struct intel_cursor_error_state {
16169                 u32 control;
16170                 u32 position;
16171                 u32 base;
16172                 u32 size;
16173         } cursor[I915_MAX_PIPES];
16174
16175         struct intel_pipe_error_state {
16176                 bool power_domain_on;
16177                 u32 source;
16178                 u32 stat;
16179         } pipe[I915_MAX_PIPES];
16180
16181         struct intel_plane_error_state {
16182                 u32 control;
16183                 u32 stride;
16184                 u32 size;
16185                 u32 pos;
16186                 u32 addr;
16187                 u32 surface;
16188                 u32 tile_offset;
16189         } plane[I915_MAX_PIPES];
16190
16191         struct intel_transcoder_error_state {
16192                 bool power_domain_on;
16193                 enum transcoder cpu_transcoder;
16194
16195                 u32 conf;
16196
16197                 u32 htotal;
16198                 u32 hblank;
16199                 u32 hsync;
16200                 u32 vtotal;
16201                 u32 vblank;
16202                 u32 vsync;
16203         } transcoder[4];
16204 };
16205
16206 struct intel_display_error_state *
16207 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16208 {
16209         struct intel_display_error_state *error;
16210         int transcoders[] = {
16211                 TRANSCODER_A,
16212                 TRANSCODER_B,
16213                 TRANSCODER_C,
16214                 TRANSCODER_EDP,
16215         };
16216         int i;
16217
16218         if (INTEL_INFO(dev_priv)->num_pipes == 0)
16219                 return NULL;
16220
16221         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16222         if (error == NULL)
16223                 return NULL;
16224
16225         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16226                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
16227
16228         for_each_pipe(dev_priv, i) {
16229                 error->pipe[i].power_domain_on =
16230                         __intel_display_power_is_enabled(dev_priv,
16231                                                          POWER_DOMAIN_PIPE(i));
16232                 if (!error->pipe[i].power_domain_on)
16233                         continue;
16234
16235                 error->cursor[i].control = I915_READ(CURCNTR(i));
16236                 error->cursor[i].position = I915_READ(CURPOS(i));
16237                 error->cursor[i].base = I915_READ(CURBASE(i));
16238
16239                 error->plane[i].control = I915_READ(DSPCNTR(i));
16240                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
16241                 if (INTEL_GEN(dev_priv) <= 3) {
16242                         error->plane[i].size = I915_READ(DSPSIZE(i));
16243                         error->plane[i].pos = I915_READ(DSPPOS(i));
16244                 }
16245                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16246                         error->plane[i].addr = I915_READ(DSPADDR(i));
16247                 if (INTEL_GEN(dev_priv) >= 4) {
16248                         error->plane[i].surface = I915_READ(DSPSURF(i));
16249                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
16250                 }
16251
16252                 error->pipe[i].source = I915_READ(PIPESRC(i));
16253
16254                 if (HAS_GMCH_DISPLAY(dev_priv))
16255                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
16256         }
16257
16258         /* Note: this does not include DSI transcoders. */
16259         error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
16260         if (HAS_DDI(dev_priv))
16261                 error->num_transcoders++; /* Account for eDP. */
16262
16263         for (i = 0; i < error->num_transcoders; i++) {
16264                 enum transcoder cpu_transcoder = transcoders[i];
16265
16266                 error->transcoder[i].power_domain_on =
16267                         __intel_display_power_is_enabled(dev_priv,
16268                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16269                 if (!error->transcoder[i].power_domain_on)
16270                         continue;
16271
16272                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16273
16274                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
16275                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
16276                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
16277                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
16278                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
16279                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
16280                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
16281         }
16282
16283         return error;
16284 }
16285
16286 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16287
16288 void
16289 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16290                                 struct intel_display_error_state *error)
16291 {
16292         struct drm_i915_private *dev_priv = m->i915;
16293         int i;
16294
16295         if (!error)
16296                 return;
16297
16298         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
16299         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16300                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16301                            error->power_well_driver);
16302         for_each_pipe(dev_priv, i) {
16303                 err_printf(m, "Pipe [%d]:\n", i);
16304                 err_printf(m, "  Power: %s\n",
16305                            onoff(error->pipe[i].power_domain_on));
16306                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16307                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16308
16309                 err_printf(m, "Plane [%d]:\n", i);
16310                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16311                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16312                 if (INTEL_GEN(dev_priv) <= 3) {
16313                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16314                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16315                 }
16316                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16317                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16318                 if (INTEL_GEN(dev_priv) >= 4) {
16319                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16320                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16321                 }
16322
16323                 err_printf(m, "Cursor [%d]:\n", i);
16324                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16325                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16326                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16327         }
16328
16329         for (i = 0; i < error->num_transcoders; i++) {
16330                 err_printf(m, "CPU transcoder: %s\n",
16331                            transcoder_name(error->transcoder[i].cpu_transcoder));
16332                 err_printf(m, "  Power: %s\n",
16333                            onoff(error->transcoder[i].power_domain_on));
16334                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16335                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16336                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16337                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16338                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16339                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16340                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16341         }
16342 }
16343
16344 #endif