drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/reservation.h>
  33 #include <linux/slab.h>
  34 #include <linux/vgaarb.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45 #include <drm/i915_drm.h>
  46
  47 #include "display/intel_crt.h"
  48 #include "display/intel_ddi.h"
  49 #include "display/intel_dp.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "i915_drv.h"
  60 #include "i915_trace.h"
  61 #include "intel_acpi.h"
  62 #include "intel_atomic.h"
  63 #include "intel_atomic_plane.h"
  64 #include "intel_bw.h"
  65 #include "intel_color.h"
  66 #include "intel_cdclk.h"
  67 #include "intel_drv.h"
  68 #include "intel_fbc.h"
  69 #include "intel_fbdev.h"
  70 #include "intel_fifo_underrun.h"
  71 #include "intel_frontbuffer.h"
  72 #include "intel_hdcp.h"
  73 #include "intel_hotplug.h"
  74 #include "intel_overlay.h"
  75 #include "intel_pipe_crc.h"
  76 #include "intel_pm.h"
  77 #include "intel_psr.h"
  78 #include "intel_quirks.h"
  79 #include "intel_sideband.h"
  80 #include "intel_sprite.h"
  81 #include "intel_tc.h"
  82
  83 /* Primary plane formats for gen <= 3 */
  84 static const u32 i8xx_primary_formats[] = {
  85         DRM_FORMAT_C8,
  86         DRM_FORMAT_RGB565,
  87         DRM_FORMAT_XRGB1555,
  88         DRM_FORMAT_XRGB8888,
  89 };
  90
  91 /* Primary plane formats for gen >= 4 */
  92 static const u32 i965_primary_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_XRGB2101010,
  98         DRM_FORMAT_XBGR2101010,
  99 };
 100
 101 static const u64 i9xx_format_modifiers[] = {
 102         I915_FORMAT_MOD_X_TILED,
 103         DRM_FORMAT_MOD_LINEAR,
 104         DRM_FORMAT_MOD_INVALID
 105 };
 106
 107 /* Cursor formats */
 108 static const u32 intel_cursor_formats[] = {
 109         DRM_FORMAT_ARGB8888,
 110 };
 111
 112 static const u64 cursor_format_modifiers[] = {
 113         DRM_FORMAT_MOD_LINEAR,
 114         DRM_FORMAT_MOD_INVALID
 115 };
 116
 117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 118                                 struct intel_crtc_state *pipe_config);
 119 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 120                                    struct intel_crtc_state *pipe_config);
 121
 122 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 123                                   struct drm_i915_gem_object *obj,
 124                                   struct drm_mode_fb_cmd2 *mode_cmd);
 125 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 126 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 127 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 128                                          const struct intel_link_m_n *m_n,
 129                                          const struct intel_link_m_n *m2_n2);
 130 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 131 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 132 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 133 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 134 static void vlv_prepare_pll(struct intel_crtc *crtc,
 135                             const struct intel_crtc_state *pipe_config);
 136 static void chv_prepare_pll(struct intel_crtc *crtc,
 137                             const struct intel_crtc_state *pipe_config);
 138 static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 139 static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 140 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 141                                     struct intel_crtc_state *crtc_state);
 142 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 143 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 144 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 145 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 146                                          struct drm_modeset_acquire_ctx *ctx);
 147 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 148
 149 struct intel_limit {
 150         struct {
 151                 int min, max;
 152         } dot, vco, n, m, m1, m2, p, p1;
 153
 154         struct {
 155                 int dot_limit;
 156                 int p2_slow, p2_fast;
 157         } p2;
 158 };
 159
 160 /* returns HPLL frequency in kHz */
 161 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 162 {
 163         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 164
 165         /* Obtain SKU information */
 166         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 167                 CCK_FUSE_HPLL_FREQ_MASK;
 168
 169         return vco_freq[hpll_freq] * 1000;
 170 }
 171
 172 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 173                       const char *name, u32 reg, int ref_freq)
 174 {
 175         u32 val;
 176         int divider;
 177
 178         val = vlv_cck_read(dev_priv, reg);
 179         divider = val & CCK_FREQUENCY_VALUES;
 180
 181         WARN((val & CCK_FREQUENCY_STATUS) !=
 182              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 183              "%s change in progress\n", name);
 184
 185         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 186 }
 187
 188 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 189                            const char *name, u32 reg)
 190 {
 191         int hpll;
 192
 193         vlv_cck_get(dev_priv);
 194
 195         if (dev_priv->hpll_freq == 0)
 196                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 197
 198         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 199
 200         vlv_cck_put(dev_priv);
 201
 202         return hpll;
 203 }
 204
 205 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 206 {
 207         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 208                 return;
 209
 210         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 211                                                       CCK_CZ_CLOCK_CONTROL);
 212
 213         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 214 }
 215
 216 static inline u32 /* units of 100MHz */
 217 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 218                     const struct intel_crtc_state *pipe_config)
 219 {
 220         if (HAS_DDI(dev_priv))
 221                 return pipe_config->port_clock; /* SPLL */
 222         else
 223                 return dev_priv->fdi_pll_freq;
 224 }
 225
 226 static const struct intel_limit intel_limits_i8xx_dac = {
 227         .dot = { .min = 25000, .max = 350000 },
 228         .vco = { .min = 908000, .max = 1512000 },
 229         .n = { .min = 2, .max = 16 },
 230         .m = { .min = 96, .max = 140 },
 231         .m1 = { .min = 18, .max = 26 },
 232         .m2 = { .min = 6, .max = 16 },
 233         .p = { .min = 4, .max = 128 },
 234         .p1 = { .min = 2, .max = 33 },
 235         .p2 = { .dot_limit = 165000,
 236                 .p2_slow = 4, .p2_fast = 2 },
 237 };
 238
 239 static const struct intel_limit intel_limits_i8xx_dvo = {
 240         .dot = { .min = 25000, .max = 350000 },
 241         .vco = { .min = 908000, .max = 1512000 },
 242         .n = { .min = 2, .max = 16 },
 243         .m = { .min = 96, .max = 140 },
 244         .m1 = { .min = 18, .max = 26 },
 245         .m2 = { .min = 6, .max = 16 },
 246         .p = { .min = 4, .max = 128 },
 247         .p1 = { .min = 2, .max = 33 },
 248         .p2 = { .dot_limit = 165000,
 249                 .p2_slow = 4, .p2_fast = 4 },
 250 };
 251
 252 static const struct intel_limit intel_limits_i8xx_lvds = {
 253         .dot = { .min = 25000, .max = 350000 },
 254         .vco = { .min = 908000, .max = 1512000 },
 255         .n = { .min = 2, .max = 16 },
 256         .m = { .min = 96, .max = 140 },
 257         .m1 = { .min = 18, .max = 26 },
 258         .m2 = { .min = 6, .max = 16 },
 259         .p = { .min = 4, .max = 128 },
 260         .p1 = { .min = 1, .max = 6 },
 261         .p2 = { .dot_limit = 165000,
 262                 .p2_slow = 14, .p2_fast = 7 },
 263 };
 264
 265 static const struct intel_limit intel_limits_i9xx_sdvo = {
 266         .dot = { .min = 20000, .max = 400000 },
 267         .vco = { .min = 1400000, .max = 2800000 },
 268         .n = { .min = 1, .max = 6 },
 269         .m = { .min = 70, .max = 120 },
 270         .m1 = { .min = 8, .max = 18 },
 271         .m2 = { .min = 3, .max = 7 },
 272         .p = { .min = 5, .max = 80 },
 273         .p1 = { .min = 1, .max = 8 },
 274         .p2 = { .dot_limit = 200000,
 275                 .p2_slow = 10, .p2_fast = 5 },
 276 };
 277
 278 static const struct intel_limit intel_limits_i9xx_lvds = {
 279         .dot = { .min = 20000, .max = 400000 },
 280         .vco = { .min = 1400000, .max = 2800000 },
 281         .n = { .min = 1, .max = 6 },
 282         .m = { .min = 70, .max = 120 },
 283         .m1 = { .min = 8, .max = 18 },
 284         .m2 = { .min = 3, .max = 7 },
 285         .p = { .min = 7, .max = 98 },
 286         .p1 = { .min = 1, .max = 8 },
 287         .p2 = { .dot_limit = 112000,
 288                 .p2_slow = 14, .p2_fast = 7 },
 289 };
 290
 291
 292 static const struct intel_limit intel_limits_g4x_sdvo = {
 293         .dot = { .min = 25000, .max = 270000 },
 294         .vco = { .min = 1750000, .max = 3500000},
 295         .n = { .min = 1, .max = 4 },
 296         .m = { .min = 104, .max = 138 },
 297         .m1 = { .min = 17, .max = 23 },
 298         .m2 = { .min = 5, .max = 11 },
 299         .p = { .min = 10, .max = 30 },
 300         .p1 = { .min = 1, .max = 3},
 301         .p2 = { .dot_limit = 270000,
 302                 .p2_slow = 10,
 303                 .p2_fast = 10
 304         },
 305 };
 306
 307 static const struct intel_limit intel_limits_g4x_hdmi = {
 308         .dot = { .min = 22000, .max = 400000 },
 309         .vco = { .min = 1750000, .max = 3500000},
 310         .n = { .min = 1, .max = 4 },
 311         .m = { .min = 104, .max = 138 },
 312         .m1 = { .min = 16, .max = 23 },
 313         .m2 = { .min = 5, .max = 11 },
 314         .p = { .min = 5, .max = 80 },
 315         .p1 = { .min = 1, .max = 8},
 316         .p2 = { .dot_limit = 165000,
 317                 .p2_slow = 10, .p2_fast = 5 },
 318 };
 319
 320 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 321         .dot = { .min = 20000, .max = 115000 },
 322         .vco = { .min = 1750000, .max = 3500000 },
 323         .n = { .min = 1, .max = 3 },
 324         .m = { .min = 104, .max = 138 },
 325         .m1 = { .min = 17, .max = 23 },
 326         .m2 = { .min = 5, .max = 11 },
 327         .p = { .min = 28, .max = 112 },
 328         .p1 = { .min = 2, .max = 8 },
 329         .p2 = { .dot_limit = 0,
 330                 .p2_slow = 14, .p2_fast = 14
 331         },
 332 };
 333
 334 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 335         .dot = { .min = 80000, .max = 224000 },
 336         .vco = { .min = 1750000, .max = 3500000 },
 337         .n = { .min = 1, .max = 3 },
 338         .m = { .min = 104, .max = 138 },
 339         .m1 = { .min = 17, .max = 23 },
 340         .m2 = { .min = 5, .max = 11 },
 341         .p = { .min = 14, .max = 42 },
 342         .p1 = { .min = 2, .max = 6 },
 343         .p2 = { .dot_limit = 0,
 344                 .p2_slow = 7, .p2_fast = 7
 345         },
 346 };
 347
 348 static const struct intel_limit intel_limits_pineview_sdvo = {
 349         .dot = { .min = 20000, .max = 400000},
 350         .vco = { .min = 1700000, .max = 3500000 },
 351         /* Pineview's Ncounter is a ring counter */
 352         .n = { .min = 3, .max = 6 },
 353         .m = { .min = 2, .max = 256 },
 354         /* Pineview only has one combined m divider, which we treat as m2. */
 355         .m1 = { .min = 0, .max = 0 },
 356         .m2 = { .min = 0, .max = 254 },
 357         .p = { .min = 5, .max = 80 },
 358         .p1 = { .min = 1, .max = 8 },
 359         .p2 = { .dot_limit = 200000,
 360                 .p2_slow = 10, .p2_fast = 5 },
 361 };
 362
 363 static const struct intel_limit intel_limits_pineview_lvds = {
 364         .dot = { .min = 20000, .max = 400000 },
 365         .vco = { .min = 1700000, .max = 3500000 },
 366         .n = { .min = 3, .max = 6 },
 367         .m = { .min = 2, .max = 256 },
 368         .m1 = { .min = 0, .max = 0 },
 369         .m2 = { .min = 0, .max = 254 },
 370         .p = { .min = 7, .max = 112 },
 371         .p1 = { .min = 1, .max = 8 },
 372         .p2 = { .dot_limit = 112000,
 373                 .p2_slow = 14, .p2_fast = 14 },
 374 };
 375
 376 /* Ironlake / Sandybridge
 377  *
 378  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 379  * the range value for them is (actual_value - 2).
 380  */
 381 static const struct intel_limit intel_limits_ironlake_dac = {
 382         .dot = { .min = 25000, .max = 350000 },
 383         .vco = { .min = 1760000, .max = 3510000 },
 384         .n = { .min = 1, .max = 5 },
 385         .m = { .min = 79, .max = 127 },
 386         .m1 = { .min = 12, .max = 22 },
 387         .m2 = { .min = 5, .max = 9 },
 388         .p = { .min = 5, .max = 80 },
 389         .p1 = { .min = 1, .max = 8 },
 390         .p2 = { .dot_limit = 225000,
 391                 .p2_slow = 10, .p2_fast = 5 },
 392 };
 393
 394 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 395         .dot = { .min = 25000, .max = 350000 },
 396         .vco = { .min = 1760000, .max = 3510000 },
 397         .n = { .min = 1, .max = 3 },
 398         .m = { .min = 79, .max = 118 },
 399         .m1 = { .min = 12, .max = 22 },
 400         .m2 = { .min = 5, .max = 9 },
 401         .p = { .min = 28, .max = 112 },
 402         .p1 = { .min = 2, .max = 8 },
 403         .p2 = { .dot_limit = 225000,
 404                 .p2_slow = 14, .p2_fast = 14 },
 405 };
 406
 407 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 408         .dot = { .min = 25000, .max = 350000 },
 409         .vco = { .min = 1760000, .max = 3510000 },
 410         .n = { .min = 1, .max = 3 },
 411         .m = { .min = 79, .max = 127 },
 412         .m1 = { .min = 12, .max = 22 },
 413         .m2 = { .min = 5, .max = 9 },
 414         .p = { .min = 14, .max = 56 },
 415         .p1 = { .min = 2, .max = 8 },
 416         .p2 = { .dot_limit = 225000,
 417                 .p2_slow = 7, .p2_fast = 7 },
 418 };
 419
 420 /* LVDS 100mhz refclk limits. */
 421 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 422         .dot = { .min = 25000, .max = 350000 },
 423         .vco = { .min = 1760000, .max = 3510000 },
 424         .n = { .min = 1, .max = 2 },
 425         .m = { .min = 79, .max = 126 },
 426         .m1 = { .min = 12, .max = 22 },
 427         .m2 = { .min = 5, .max = 9 },
 428         .p = { .min = 28, .max = 112 },
 429         .p1 = { .min = 2, .max = 8 },
 430         .p2 = { .dot_limit = 225000,
 431                 .p2_slow = 14, .p2_fast = 14 },
 432 };
 433
 434 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 435         .dot = { .min = 25000, .max = 350000 },
 436         .vco = { .min = 1760000, .max = 3510000 },
 437         .n = { .min = 1, .max = 3 },
 438         .m = { .min = 79, .max = 126 },
 439         .m1 = { .min = 12, .max = 22 },
 440         .m2 = { .min = 5, .max = 9 },
 441         .p = { .min = 14, .max = 42 },
 442         .p1 = { .min = 2, .max = 6 },
 443         .p2 = { .dot_limit = 225000,
 444                 .p2_slow = 7, .p2_fast = 7 },
 445 };
 446
 447 static const struct intel_limit intel_limits_vlv = {
 448          /*
 449           * These are the data rate limits (measured in fast clocks)
 450           * since those are the strictest limits we have. The fast
 451           * clock and actual rate limits are more relaxed, so checking
 452           * them would make no difference.
 453           */
 454         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 455         .vco = { .min = 4000000, .max = 6000000 },
 456         .n = { .min = 1, .max = 7 },
 457         .m1 = { .min = 2, .max = 3 },
 458         .m2 = { .min = 11, .max = 156 },
 459         .p1 = { .min = 2, .max = 3 },
 460         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 461 };
 462
 463 static const struct intel_limit intel_limits_chv = {
 464         /*
 465          * These are the data rate limits (measured in fast clocks)
 466          * since those are the strictest limits we have.  The fast
 467          * clock and actual rate limits are more relaxed, so checking
 468          * them would make no difference.
 469          */
 470         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 471         .vco = { .min = 4800000, .max = 6480000 },
 472         .n = { .min = 1, .max = 1 },
 473         .m1 = { .min = 2, .max = 2 },
 474         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 475         .p1 = { .min = 2, .max = 4 },
 476         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 477 };
 478
 479 static const struct intel_limit intel_limits_bxt = {
 480         /* FIXME: find real dot limits */
 481         .dot = { .min = 0, .max = INT_MAX },
 482         .vco = { .min = 4800000, .max = 6700000 },
 483         .n = { .min = 1, .max = 1 },
 484         .m1 = { .min = 2, .max = 2 },
 485         /* FIXME: find real m2 limits */
 486         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 487         .p1 = { .min = 2, .max = 4 },
 488         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 489 };
 490
 491 /* WA Display #0827: Gen9:all */
 492 static void
 493 skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable)
 494 {
 495         if (enable)
 496                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 497                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 498                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 499         else
 500                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 501                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 502                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 503 }
 504
 505 /* Wa_2006604312:icl */
 506 static void
 507 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 508                        bool enable)
 509 {
 510         if (enable)
 511                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 512                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 513         else
 514                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 515                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 516 }
 517
 518 static bool
 519 needs_modeset(const struct intel_crtc_state *state)
 520 {
 521         return drm_atomic_crtc_needs_modeset(&state->base);
 522 }
 523
 524 /*
 525  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 526  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 527  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 528  * The helpers' return value is the rate of the clock that is fed to the
 529  * display engine's pipe which can be the above fast dot clock rate or a
 530  * divided-down version of it.
 531  */
 532 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 533 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 534 {
 535         clock->m = clock->m2 + 2;
 536         clock->p = clock->p1 * clock->p2;
 537         if (WARN_ON(clock->n == 0 || clock->p == 0))
 538                 return 0;
 539         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 540         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 541
 542         return clock->dot;
 543 }
 544
 545 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 546 {
 547         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 548 }
 549
 550 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = i9xx_dpll_compute_m(clock);
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 563 {
 564         clock->m = clock->m1 * clock->m2;
 565         clock->p = clock->p1 * clock->p2;
 566         if (WARN_ON(clock->n == 0 || clock->p == 0))
 567                 return 0;
 568         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 569         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 570
 571         return clock->dot / 5;
 572 }
 573
 574 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 575 {
 576         clock->m = clock->m1 * clock->m2;
 577         clock->p = clock->p1 * clock->p2;
 578         if (WARN_ON(clock->n == 0 || clock->p == 0))
 579                 return 0;
 580         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 581                                            clock->n << 22);
 582         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 583
 584         return clock->dot / 5;
 585 }
 586
 587 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 588
 589 /*
 590  * Returns whether the given set of divisors are valid for a given refclk with
 591  * the given connectors.
 592  */
 593 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 594                                const struct intel_limit *limit,
 595                                const struct dpll *clock)
 596 {
 597         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 598                 INTELPllInvalid("n out of range\n");
 599         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 600                 INTELPllInvalid("p1 out of range\n");
 601         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 602                 INTELPllInvalid("m2 out of range\n");
 603         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 604                 INTELPllInvalid("m1 out of range\n");
 605
 606         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 607             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 608                 if (clock->m1 <= clock->m2)
 609                         INTELPllInvalid("m1 <= m2\n");
 610
 611         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 612             !IS_GEN9_LP(dev_priv)) {
 613                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 614                         INTELPllInvalid("p out of range\n");
 615                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 616                         INTELPllInvalid("m out of range\n");
 617         }
 618
 619         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 620                 INTELPllInvalid("vco out of range\n");
 621         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 622          * connector, etc., rather than just a single range.
 623          */
 624         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 625                 INTELPllInvalid("dot out of range\n");
 626
 627         return true;
 628 }
 629
 630 static int
 631 i9xx_select_p2_div(const struct intel_limit *limit,
 632                    const struct intel_crtc_state *crtc_state,
 633                    int target)
 634 {
 635         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 636
 637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 638                 /*
 639                  * For LVDS just rely on its current settings for dual-channel.
 640                  * We haven't figured out how to reliably set up different
 641                  * single/dual channel state, if we even can.
 642                  */
 643                 if (intel_is_dual_link_lvds(dev_priv))
 644                         return limit->p2.p2_fast;
 645                 else
 646                         return limit->p2.p2_slow;
 647         } else {
 648                 if (target < limit->p2.dot_limit)
 649                         return limit->p2.p2_slow;
 650                 else
 651                         return limit->p2.p2_fast;
 652         }
 653 }
 654
 655 /*
 656  * Returns a set of divisors for the desired target clock with the given
 657  * refclk, or FALSE.  The returned values represent the clock equation:
 658  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 659  *
 660  * Target and reference clocks are specified in kHz.
 661  *
 662  * If match_clock is provided, then best_clock P divider must match the P
 663  * divider from @match_clock used for LVDS downclocking.
 664  */
 665 static bool
 666 i9xx_find_best_dpll(const struct intel_limit *limit,
 667                     struct intel_crtc_state *crtc_state,
 668                     int target, int refclk, struct dpll *match_clock,
 669                     struct dpll *best_clock)
 670 {
 671         struct drm_device *dev = crtc_state->base.crtc->dev;
 672         struct dpll clock;
 673         int err = target;
 674
 675         memset(best_clock, 0, sizeof(*best_clock));
 676
 677         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 678
 679         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 680              clock.m1++) {
 681                 for (clock.m2 = limit->m2.min;
 682                      clock.m2 <= limit->m2.max; clock.m2++) {
 683                         if (clock.m2 >= clock.m1)
 684                                 break;
 685                         for (clock.n = limit->n.min;
 686                              clock.n <= limit->n.max; clock.n++) {
 687                                 for (clock.p1 = limit->p1.min;
 688                                         clock.p1 <= limit->p1.max; clock.p1++) {
 689                                         int this_err;
 690
 691                                         i9xx_calc_dpll_params(refclk, &clock);
 692                                         if (!intel_PLL_is_valid(to_i915(dev),
 693                                                                 limit,
 694                                                                 &clock))
 695                                                 continue;
 696                                         if (match_clock &&
 697                                             clock.p != match_clock->p)
 698                                                 continue;
 699
 700                                         this_err = abs(clock.dot - target);
 701                                         if (this_err < err) {
 702                                                 *best_clock = clock;
 703                                                 err = this_err;
 704                                         }
 705                                 }
 706                         }
 707                 }
 708         }
 709
 710         return (err != target);
 711 }
 712
 713 /*
 714  * Returns a set of divisors for the desired target clock with the given
 715  * refclk, or FALSE.  The returned values represent the clock equation:
 716  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 717  *
 718  * Target and reference clocks are specified in kHz.
 719  *
 720  * If match_clock is provided, then best_clock P divider must match the P
 721  * divider from @match_clock used for LVDS downclocking.
 722  */
 723 static bool
 724 pnv_find_best_dpll(const struct intel_limit *limit,
 725                    struct intel_crtc_state *crtc_state,
 726                    int target, int refclk, struct dpll *match_clock,
 727                    struct dpll *best_clock)
 728 {
 729         struct drm_device *dev = crtc_state->base.crtc->dev;
 730         struct dpll clock;
 731         int err = target;
 732
 733         memset(best_clock, 0, sizeof(*best_clock));
 734
 735         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 736
 737         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 738              clock.m1++) {
 739                 for (clock.m2 = limit->m2.min;
 740                      clock.m2 <= limit->m2.max; clock.m2++) {
 741                         for (clock.n = limit->n.min;
 742                              clock.n <= limit->n.max; clock.n++) {
 743                                 for (clock.p1 = limit->p1.min;
 744                                         clock.p1 <= limit->p1.max; clock.p1++) {
 745                                         int this_err;
 746
 747                                         pnv_calc_dpll_params(refclk, &clock);
 748                                         if (!intel_PLL_is_valid(to_i915(dev),
 749                                                                 limit,
 750                                                                 &clock))
 751                                                 continue;
 752                                         if (match_clock &&
 753                                             clock.p != match_clock->p)
 754                                                 continue;
 755
 756                                         this_err = abs(clock.dot - target);
 757                                         if (this_err < err) {
 758                                                 *best_clock = clock;
 759                                                 err = this_err;
 760                                         }
 761                                 }
 762                         }
 763                 }
 764         }
 765
 766         return (err != target);
 767 }
 768
 769 /*
 770  * Returns a set of divisors for the desired target clock with the given
 771  * refclk, or FALSE.  The returned values represent the clock equation:
 772  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 773  *
 774  * Target and reference clocks are specified in kHz.
 775  *
 776  * If match_clock is provided, then best_clock P divider must match the P
 777  * divider from @match_clock used for LVDS downclocking.
 778  */
 779 static bool
 780 g4x_find_best_dpll(const struct intel_limit *limit,
 781                    struct intel_crtc_state *crtc_state,
 782                    int target, int refclk, struct dpll *match_clock,
 783                    struct dpll *best_clock)
 784 {
 785         struct drm_device *dev = crtc_state->base.crtc->dev;
 786         struct dpll clock;
 787         int max_n;
 788         bool found = false;
 789         /* approximately equals target * 0.00585 */
 790         int err_most = (target >> 8) + (target >> 9);
 791
 792         memset(best_clock, 0, sizeof(*best_clock));
 793
 794         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 795
 796         max_n = limit->n.max;
 797         /* based on hardware requirement, prefer smaller n to precision */
 798         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 799                 /* based on hardware requirement, prefere larger m1,m2 */
 800                 for (clock.m1 = limit->m1.max;
 801                      clock.m1 >= limit->m1.min; clock.m1--) {
 802                         for (clock.m2 = limit->m2.max;
 803                              clock.m2 >= limit->m2.min; clock.m2--) {
 804                                 for (clock.p1 = limit->p1.max;
 805                                      clock.p1 >= limit->p1.min; clock.p1--) {
 806                                         int this_err;
 807
 808                                         i9xx_calc_dpll_params(refclk, &clock);
 809                                         if (!intel_PLL_is_valid(to_i915(dev),
 810                                                                 limit,
 811                                                                 &clock))
 812                                                 continue;
 813
 814                                         this_err = abs(clock.dot - target);
 815                                         if (this_err < err_most) {
 816                                                 *best_clock = clock;
 817                                                 err_most = this_err;
 818                                                 max_n = clock.n;
 819                                                 found = true;
 820                                         }
 821                                 }
 822                         }
 823                 }
 824         }
 825         return found;
 826 }
 827
 828 /*
 829  * Check if the calculated PLL configuration is more optimal compared to the
 830  * best configuration and error found so far. Return the calculated error.
 831  */
 832 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 833                                const struct dpll *calculated_clock,
 834                                const struct dpll *best_clock,
 835                                unsigned int best_error_ppm,
 836                                unsigned int *error_ppm)
 837 {
 838         /*
 839          * For CHV ignore the error and consider only the P value.
 840          * Prefer a bigger P value based on HW requirements.
 841          */
 842         if (IS_CHERRYVIEW(to_i915(dev))) {
 843                 *error_ppm = 0;
 844
 845                 return calculated_clock->p > best_clock->p;
 846         }
 847
 848         if (WARN_ON_ONCE(!target_freq))
 849                 return false;
 850
 851         *error_ppm = div_u64(1000000ULL *
 852                                 abs(target_freq - calculated_clock->dot),
 853                              target_freq);
 854         /*
 855          * Prefer a better P value over a better (smaller) error if the error
 856          * is small. Ensure this preference for future configurations too by
 857          * setting the error to 0.
 858          */
 859         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 860                 *error_ppm = 0;
 861
 862                 return true;
 863         }
 864
 865         return *error_ppm + 10 < best_error_ppm;
 866 }
 867
 868 /*
 869  * Returns a set of divisors for the desired target clock with the given
 870  * refclk, or FALSE.  The returned values represent the clock equation:
 871  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 872  */
 873 static bool
 874 vlv_find_best_dpll(const struct intel_limit *limit,
 875                    struct intel_crtc_state *crtc_state,
 876                    int target, int refclk, struct dpll *match_clock,
 877                    struct dpll *best_clock)
 878 {
 879         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 880         struct drm_device *dev = crtc->base.dev;
 881         struct dpll clock;
 882         unsigned int bestppm = 1000000;
 883         /* min update 19.2 MHz */
 884         int max_n = min(limit->n.max, refclk / 19200);
 885         bool found = false;
 886
 887         target *= 5; /* fast clock */
 888
 889         memset(best_clock, 0, sizeof(*best_clock));
 890
 891         /* based on hardware requirement, prefer smaller n to precision */
 892         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 893                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 894                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 895                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 896                                 clock.p = clock.p1 * clock.p2;
 897                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 898                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 899                                         unsigned int ppm;
 900
 901                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 902                                                                      refclk * clock.m1);
 903
 904                                         vlv_calc_dpll_params(refclk, &clock);
 905
 906                                         if (!intel_PLL_is_valid(to_i915(dev),
 907                                                                 limit,
 908                                                                 &clock))
 909                                                 continue;
 910
 911                                         if (!vlv_PLL_is_optimal(dev, target,
 912                                                                 &clock,
 913                                                                 best_clock,
 914                                                                 bestppm, &ppm))
 915                                                 continue;
 916
 917                                         *best_clock = clock;
 918                                         bestppm = ppm;
 919                                         found = true;
 920                                 }
 921                         }
 922                 }
 923         }
 924
 925         return found;
 926 }
 927
 928 /*
 929  * Returns a set of divisors for the desired target clock with the given
 930  * refclk, or FALSE.  The returned values represent the clock equation:
 931  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 932  */
 933 static bool
 934 chv_find_best_dpll(const struct intel_limit *limit,
 935                    struct intel_crtc_state *crtc_state,
 936                    int target, int refclk, struct dpll *match_clock,
 937                    struct dpll *best_clock)
 938 {
 939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 940         struct drm_device *dev = crtc->base.dev;
 941         unsigned int best_error_ppm;
 942         struct dpll clock;
 943         u64 m2;
 944         int found = false;
 945
 946         memset(best_clock, 0, sizeof(*best_clock));
 947         best_error_ppm = 1000000;
 948
 949         /*
 950          * Based on hardware doc, the n always set to 1, and m1 always
 951          * set to 2.  If requires to support 200Mhz refclk, we need to
 952          * revisit this because n may not 1 anymore.
 953          */
 954         clock.n = 1, clock.m1 = 2;
 955         target *= 5;    /* fast clock */
 956
 957         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 958                 for (clock.p2 = limit->p2.p2_fast;
 959                                 clock.p2 >= limit->p2.p2_slow;
 960                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 961                         unsigned int error_ppm;
 962
 963                         clock.p = clock.p1 * clock.p2;
 964
 965                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
 966                                                    refclk * clock.m1);
 967
 968                         if (m2 > INT_MAX/clock.m1)
 969                                 continue;
 970
 971                         clock.m2 = m2;
 972
 973                         chv_calc_dpll_params(refclk, &clock);
 974
 975                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 976                                 continue;
 977
 978                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 979                                                 best_error_ppm, &error_ppm))
 980                                 continue;
 981
 982                         *best_clock = clock;
 983                         best_error_ppm = error_ppm;
 984                         found = true;
 985                 }
 986         }
 987
 988         return found;
 989 }
 990
 991 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 992                         struct dpll *best_clock)
 993 {
 994         int refclk = 100000;
 995         const struct intel_limit *limit = &intel_limits_bxt;
 996
 997         return chv_find_best_dpll(limit, crtc_state,
 998                                   crtc_state->port_clock, refclk,
 999                                   NULL, best_clock);
1000 }
1001
1002 bool intel_crtc_active(struct intel_crtc *crtc)
1003 {
1004         /* Be paranoid as we can arrive here with only partial
1005          * state retrieved from the hardware during setup.
1006          *
1007          * We can ditch the adjusted_mode.crtc_clock check as soon
1008          * as Haswell has gained clock readout/fastboot support.
1009          *
1010          * We can ditch the crtc->primary->state->fb check as soon as we can
1011          * properly reconstruct framebuffers.
1012          *
1013          * FIXME: The intel_crtc->active here should be switched to
1014          * crtc->state->active once we have proper CRTC states wired up
1015          * for atomic.
1016          */
1017         return crtc->active && crtc->base.primary->state->fb &&
1018                 crtc->config->base.adjusted_mode.crtc_clock;
1019 }
1020
1021 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1022                                              enum pipe pipe)
1023 {
1024         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1025
1026         return crtc->config->cpu_transcoder;
1027 }
1028
1029 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1030                                     enum pipe pipe)
1031 {
1032         i915_reg_t reg = PIPEDSL(pipe);
1033         u32 line1, line2;
1034         u32 line_mask;
1035
1036         if (IS_GEN(dev_priv, 2))
1037                 line_mask = DSL_LINEMASK_GEN2;
1038         else
1039                 line_mask = DSL_LINEMASK_GEN3;
1040
1041         line1 = I915_READ(reg) & line_mask;
1042         msleep(5);
1043         line2 = I915_READ(reg) & line_mask;
1044
1045         return line1 != line2;
1046 }
1047
1048 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1049 {
1050         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1051         enum pipe pipe = crtc->pipe;
1052
1053         /* Wait for the display line to settle/start moving */
1054         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1055                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1056                           pipe_name(pipe), onoff(state));
1057 }
1058
1059 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1060 {
1061         wait_for_pipe_scanline_moving(crtc, false);
1062 }
1063
1064 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1065 {
1066         wait_for_pipe_scanline_moving(crtc, true);
1067 }
1068
1069 static void
1070 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1071 {
1072         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1073         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1074
1075         if (INTEL_GEN(dev_priv) >= 4) {
1076                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1077                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1078
1079                 /* Wait for the Pipe State to go off */
1080                 if (intel_wait_for_register(&dev_priv->uncore,
1081                                             reg, I965_PIPECONF_ACTIVE, 0,
1082                                             100))
1083                         WARN(1, "pipe_off wait timed out\n");
1084         } else {
1085                 intel_wait_for_pipe_scanline_stopped(crtc);
1086         }
1087 }
1088
1089 /* Only for pre-ILK configs */
1090 void assert_pll(struct drm_i915_private *dev_priv,
1091                 enum pipe pipe, bool state)
1092 {
1093         u32 val;
1094         bool cur_state;
1095
1096         val = I915_READ(DPLL(pipe));
1097         cur_state = !!(val & DPLL_VCO_ENABLE);
1098         I915_STATE_WARN(cur_state != state,
1099              "PLL state assertion failure (expected %s, current %s)\n",
1100                         onoff(state), onoff(cur_state));
1101 }
1102
1103 /* XXX: the dsi pll is shared between MIPI DSI ports */
1104 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1105 {
1106         u32 val;
1107         bool cur_state;
1108
1109         vlv_cck_get(dev_priv);
1110         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1111         vlv_cck_put(dev_priv);
1112
1113         cur_state = val & DSI_PLL_VCO_EN;
1114         I915_STATE_WARN(cur_state != state,
1115              "DSI PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1120                           enum pipe pipe, bool state)
1121 {
1122         bool cur_state;
1123         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1124                                                                       pipe);
1125
1126         if (HAS_DDI(dev_priv)) {
1127                 /* DDI does not have a specific FDI_TX register */
1128                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1129                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1130         } else {
1131                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1132                 cur_state = !!(val & FDI_TX_ENABLE);
1133         }
1134         I915_STATE_WARN(cur_state != state,
1135              "FDI TX state assertion failure (expected %s, current %s)\n",
1136                         onoff(state), onoff(cur_state));
1137 }
1138 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1139 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1140
1141 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1142                           enum pipe pipe, bool state)
1143 {
1144         u32 val;
1145         bool cur_state;
1146
1147         val = I915_READ(FDI_RX_CTL(pipe));
1148         cur_state = !!(val & FDI_RX_ENABLE);
1149         I915_STATE_WARN(cur_state != state,
1150              "FDI RX state assertion failure (expected %s, current %s)\n",
1151                         onoff(state), onoff(cur_state));
1152 }
1153 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1154 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1155
1156 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1157                                       enum pipe pipe)
1158 {
1159         u32 val;
1160
1161         /* ILK FDI PLL is always enabled */
1162         if (IS_GEN(dev_priv, 5))
1163                 return;
1164
1165         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1166         if (HAS_DDI(dev_priv))
1167                 return;
1168
1169         val = I915_READ(FDI_TX_CTL(pipe));
1170         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1171 }
1172
1173 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1174                        enum pipe pipe, bool state)
1175 {
1176         u32 val;
1177         bool cur_state;
1178
1179         val = I915_READ(FDI_RX_CTL(pipe));
1180         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1181         I915_STATE_WARN(cur_state != state,
1182              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1183                         onoff(state), onoff(cur_state));
1184 }
1185
1186 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1187 {
1188         i915_reg_t pp_reg;
1189         u32 val;
1190         enum pipe panel_pipe = INVALID_PIPE;
1191         bool locked = true;
1192
1193         if (WARN_ON(HAS_DDI(dev_priv)))
1194                 return;
1195
1196         if (HAS_PCH_SPLIT(dev_priv)) {
1197                 u32 port_sel;
1198
1199                 pp_reg = PP_CONTROL(0);
1200                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1201
1202                 switch (port_sel) {
1203                 case PANEL_PORT_SELECT_LVDS:
1204                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1205                         break;
1206                 case PANEL_PORT_SELECT_DPA:
1207                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1208                         break;
1209                 case PANEL_PORT_SELECT_DPC:
1210                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1211                         break;
1212                 case PANEL_PORT_SELECT_DPD:
1213                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1214                         break;
1215                 default:
1216                         MISSING_CASE(port_sel);
1217                         break;
1218                 }
1219         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1220                 /* presumably write lock depends on pipe, not port select */
1221                 pp_reg = PP_CONTROL(pipe);
1222                 panel_pipe = pipe;
1223         } else {
1224                 u32 port_sel;
1225
1226                 pp_reg = PP_CONTROL(0);
1227                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1228
1229                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1230                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1231         }
1232
1233         val = I915_READ(pp_reg);
1234         if (!(val & PANEL_POWER_ON) ||
1235             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1236                 locked = false;
1237
1238         I915_STATE_WARN(panel_pipe == pipe && locked,
1239              "panel assertion failure, pipe %c regs locked\n",
1240              pipe_name(pipe));
1241 }
1242
1243 void assert_pipe(struct drm_i915_private *dev_priv,
1244                  enum pipe pipe, bool state)
1245 {
1246         bool cur_state;
1247         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1248                                                                       pipe);
1249         enum intel_display_power_domain power_domain;
1250         intel_wakeref_t wakeref;
1251
1252         /* we keep both pipes enabled on 830 */
1253         if (IS_I830(dev_priv))
1254                 state = true;
1255
1256         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1257         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1258         if (wakeref) {
1259                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1260                 cur_state = !!(val & PIPECONF_ENABLE);
1261
1262                 intel_display_power_put(dev_priv, power_domain, wakeref);
1263         } else {
1264                 cur_state = false;
1265         }
1266
1267         I915_STATE_WARN(cur_state != state,
1268              "pipe %c assertion failure (expected %s, current %s)\n",
1269                         pipe_name(pipe), onoff(state), onoff(cur_state));
1270 }
1271
1272 static void assert_plane(struct intel_plane *plane, bool state)
1273 {
1274         enum pipe pipe;
1275         bool cur_state;
1276
1277         cur_state = plane->get_hw_state(plane, &pipe);
1278
1279         I915_STATE_WARN(cur_state != state,
1280                         "%s assertion failure (expected %s, current %s)\n",
1281                         plane->base.name, onoff(state), onoff(cur_state));
1282 }
1283
1284 #define assert_plane_enabled(p) assert_plane(p, true)
1285 #define assert_plane_disabled(p) assert_plane(p, false)
1286
1287 static void assert_planes_disabled(struct intel_crtc *crtc)
1288 {
1289         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1290         struct intel_plane *plane;
1291
1292         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1293                 assert_plane_disabled(plane);
1294 }
1295
1296 static void assert_vblank_disabled(struct drm_crtc *crtc)
1297 {
1298         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1299                 drm_crtc_vblank_put(crtc);
1300 }
1301
1302 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1303                                     enum pipe pipe)
1304 {
1305         u32 val;
1306         bool enabled;
1307
1308         val = I915_READ(PCH_TRANSCONF(pipe));
1309         enabled = !!(val & TRANS_ENABLE);
1310         I915_STATE_WARN(enabled,
1311              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1312              pipe_name(pipe));
1313 }
1314
1315 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1316                                    enum pipe pipe, enum port port,
1317                                    i915_reg_t dp_reg)
1318 {
1319         enum pipe port_pipe;
1320         bool state;
1321
1322         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1323
1324         I915_STATE_WARN(state && port_pipe == pipe,
1325                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1326                         port_name(port), pipe_name(pipe));
1327
1328         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1329                         "IBX PCH DP %c still using transcoder B\n",
1330                         port_name(port));
1331 }
1332
1333 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1334                                      enum pipe pipe, enum port port,
1335                                      i915_reg_t hdmi_reg)
1336 {
1337         enum pipe port_pipe;
1338         bool state;
1339
1340         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1341
1342         I915_STATE_WARN(state && port_pipe == pipe,
1343                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1344                         port_name(port), pipe_name(pipe));
1345
1346         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1347                         "IBX PCH HDMI %c still using transcoder B\n",
1348                         port_name(port));
1349 }
1350
1351 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1352                                       enum pipe pipe)
1353 {
1354         enum pipe port_pipe;
1355
1356         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1357         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1358         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1359
1360         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1361                         port_pipe == pipe,
1362                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1363                         pipe_name(pipe));
1364
1365         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1366                         port_pipe == pipe,
1367                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1368                         pipe_name(pipe));
1369
1370         /* PCH SDVOB multiplex with HDMIB */
1371         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1372         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1373         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1374 }
1375
1376 static void _vlv_enable_pll(struct intel_crtc *crtc,
1377                             const struct intel_crtc_state *pipe_config)
1378 {
1379         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1380         enum pipe pipe = crtc->pipe;
1381
1382         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1383         POSTING_READ(DPLL(pipe));
1384         udelay(150);
1385
1386         if (intel_wait_for_register(&dev_priv->uncore,
1387                                     DPLL(pipe),
1388                                     DPLL_LOCK_VLV,
1389                                     DPLL_LOCK_VLV,
1390                                     1))
1391                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1392 }
1393
1394 static void vlv_enable_pll(struct intel_crtc *crtc,
1395                            const struct intel_crtc_state *pipe_config)
1396 {
1397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1398         enum pipe pipe = crtc->pipe;
1399
1400         assert_pipe_disabled(dev_priv, pipe);
1401
1402         /* PLL is protected by panel, make sure we can write it */
1403         assert_panel_unlocked(dev_priv, pipe);
1404
1405         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1406                 _vlv_enable_pll(crtc, pipe_config);
1407
1408         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1409         POSTING_READ(DPLL_MD(pipe));
1410 }
1411
1412
1413 static void _chv_enable_pll(struct intel_crtc *crtc,
1414                             const struct intel_crtc_state *pipe_config)
1415 {
1416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1417         enum pipe pipe = crtc->pipe;
1418         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1419         u32 tmp;
1420
1421         vlv_dpio_get(dev_priv);
1422
1423         /* Enable back the 10bit clock to display controller */
1424         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1425         tmp |= DPIO_DCLKP_EN;
1426         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1427
1428         vlv_dpio_put(dev_priv);
1429
1430         /*
1431          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1432          */
1433         udelay(1);
1434
1435         /* Enable PLL */
1436         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1437
1438         /* Check PLL is locked */
1439         if (intel_wait_for_register(&dev_priv->uncore,
1440                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1441                                     1))
1442                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1443 }
1444
1445 static void chv_enable_pll(struct intel_crtc *crtc,
1446                            const struct intel_crtc_state *pipe_config)
1447 {
1448         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1449         enum pipe pipe = crtc->pipe;
1450
1451         assert_pipe_disabled(dev_priv, pipe);
1452
1453         /* PLL is protected by panel, make sure we can write it */
1454         assert_panel_unlocked(dev_priv, pipe);
1455
1456         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1457                 _chv_enable_pll(crtc, pipe_config);
1458
1459         if (pipe != PIPE_A) {
1460                 /*
1461                  * WaPixelRepeatModeFixForC0:chv
1462                  *
1463                  * DPLLCMD is AWOL. Use chicken bits to propagate
1464                  * the value from DPLLBMD to either pipe B or C.
1465                  */
1466                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1467                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1468                 I915_WRITE(CBR4_VLV, 0);
1469                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1470
1471                 /*
1472                  * DPLLB VGA mode also seems to cause problems.
1473                  * We should always have it disabled.
1474                  */
1475                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1476         } else {
1477                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1478                 POSTING_READ(DPLL_MD(pipe));
1479         }
1480 }
1481
1482 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1483 {
1484         if (IS_I830(dev_priv))
1485                 return false;
1486
1487         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1488 }
1489
1490 static void i9xx_enable_pll(struct intel_crtc *crtc,
1491                             const struct intel_crtc_state *crtc_state)
1492 {
1493         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1494         i915_reg_t reg = DPLL(crtc->pipe);
1495         u32 dpll = crtc_state->dpll_hw_state.dpll;
1496         int i;
1497
1498         assert_pipe_disabled(dev_priv, crtc->pipe);
1499
1500         /* PLL is protected by panel, make sure we can write it */
1501         if (i9xx_has_pps(dev_priv))
1502                 assert_panel_unlocked(dev_priv, crtc->pipe);
1503
1504         /*
1505          * Apparently we need to have VGA mode enabled prior to changing
1506          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1507          * dividers, even though the register value does change.
1508          */
1509         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1510         I915_WRITE(reg, dpll);
1511
1512         /* Wait for the clocks to stabilize. */
1513         POSTING_READ(reg);
1514         udelay(150);
1515
1516         if (INTEL_GEN(dev_priv) >= 4) {
1517                 I915_WRITE(DPLL_MD(crtc->pipe),
1518                            crtc_state->dpll_hw_state.dpll_md);
1519         } else {
1520                 /* The pixel multiplier can only be updated once the
1521                  * DPLL is enabled and the clocks are stable.
1522                  *
1523                  * So write it again.
1524                  */
1525                 I915_WRITE(reg, dpll);
1526         }
1527
1528         /* We do this three times for luck */
1529         for (i = 0; i < 3; i++) {
1530                 I915_WRITE(reg, dpll);
1531                 POSTING_READ(reg);
1532                 udelay(150); /* wait for warmup */
1533         }
1534 }
1535
1536 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1537 {
1538         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1539         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1540         enum pipe pipe = crtc->pipe;
1541
1542         /* Don't disable pipe or pipe PLLs if needed */
1543         if (IS_I830(dev_priv))
1544                 return;
1545
1546         /* Make sure the pipe isn't still relying on us */
1547         assert_pipe_disabled(dev_priv, pipe);
1548
1549         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1550         POSTING_READ(DPLL(pipe));
1551 }
1552
1553 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1554 {
1555         u32 val;
1556
1557         /* Make sure the pipe isn't still relying on us */
1558         assert_pipe_disabled(dev_priv, pipe);
1559
1560         val = DPLL_INTEGRATED_REF_CLK_VLV |
1561                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1562         if (pipe != PIPE_A)
1563                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1564
1565         I915_WRITE(DPLL(pipe), val);
1566         POSTING_READ(DPLL(pipe));
1567 }
1568
1569 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1570 {
1571         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1572         u32 val;
1573
1574         /* Make sure the pipe isn't still relying on us */
1575         assert_pipe_disabled(dev_priv, pipe);
1576
1577         val = DPLL_SSC_REF_CLK_CHV |
1578                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1579         if (pipe != PIPE_A)
1580                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1581
1582         I915_WRITE(DPLL(pipe), val);
1583         POSTING_READ(DPLL(pipe));
1584
1585         vlv_dpio_get(dev_priv);
1586
1587         /* Disable 10bit clock to display controller */
1588         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1589         val &= ~DPIO_DCLKP_EN;
1590         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1591
1592         vlv_dpio_put(dev_priv);
1593 }
1594
1595 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1596                          struct intel_digital_port *dport,
1597                          unsigned int expected_mask)
1598 {
1599         u32 port_mask;
1600         i915_reg_t dpll_reg;
1601
1602         switch (dport->base.port) {
1603         case PORT_B:
1604                 port_mask = DPLL_PORTB_READY_MASK;
1605                 dpll_reg = DPLL(0);
1606                 break;
1607         case PORT_C:
1608                 port_mask = DPLL_PORTC_READY_MASK;
1609                 dpll_reg = DPLL(0);
1610                 expected_mask <<= 4;
1611                 break;
1612         case PORT_D:
1613                 port_mask = DPLL_PORTD_READY_MASK;
1614                 dpll_reg = DPIO_PHY_STATUS;
1615                 break;
1616         default:
1617                 BUG();
1618         }
1619
1620         if (intel_wait_for_register(&dev_priv->uncore,
1621                                     dpll_reg, port_mask, expected_mask,
1622                                     1000))
1623                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1624                      port_name(dport->base.port),
1625                      I915_READ(dpll_reg) & port_mask, expected_mask);
1626 }
1627
1628 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1629 {
1630         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1631         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1632         enum pipe pipe = crtc->pipe;
1633         i915_reg_t reg;
1634         u32 val, pipeconf_val;
1635
1636         /* Make sure PCH DPLL is enabled */
1637         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1638
1639         /* FDI must be feeding us bits for PCH ports */
1640         assert_fdi_tx_enabled(dev_priv, pipe);
1641         assert_fdi_rx_enabled(dev_priv, pipe);
1642
1643         if (HAS_PCH_CPT(dev_priv)) {
1644                 /* Workaround: Set the timing override bit before enabling the
1645                  * pch transcoder. */
1646                 reg = TRANS_CHICKEN2(pipe);
1647                 val = I915_READ(reg);
1648                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1649                 I915_WRITE(reg, val);
1650         }
1651
1652         reg = PCH_TRANSCONF(pipe);
1653         val = I915_READ(reg);
1654         pipeconf_val = I915_READ(PIPECONF(pipe));
1655
1656         if (HAS_PCH_IBX(dev_priv)) {
1657                 /*
1658                  * Make the BPC in transcoder be consistent with
1659                  * that in pipeconf reg. For HDMI we must use 8bpc
1660                  * here for both 8bpc and 12bpc.
1661                  */
1662                 val &= ~PIPECONF_BPC_MASK;
1663                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1664                         val |= PIPECONF_8BPC;
1665                 else
1666                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1667         }
1668
1669         val &= ~TRANS_INTERLACE_MASK;
1670         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1671                 if (HAS_PCH_IBX(dev_priv) &&
1672                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1673                         val |= TRANS_LEGACY_INTERLACED_ILK;
1674                 else
1675                         val |= TRANS_INTERLACED;
1676         } else {
1677                 val |= TRANS_PROGRESSIVE;
1678         }
1679
1680         I915_WRITE(reg, val | TRANS_ENABLE);
1681         if (intel_wait_for_register(&dev_priv->uncore,
1682                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1683                                     100))
1684                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1685 }
1686
1687 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1688                                       enum transcoder cpu_transcoder)
1689 {
1690         u32 val, pipeconf_val;
1691
1692         /* FDI must be feeding us bits for PCH ports */
1693         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1694         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1695
1696         /* Workaround: set timing override bit. */
1697         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1698         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1699         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1700
1701         val = TRANS_ENABLE;
1702         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1703
1704         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1705             PIPECONF_INTERLACED_ILK)
1706                 val |= TRANS_INTERLACED;
1707         else
1708                 val |= TRANS_PROGRESSIVE;
1709
1710         I915_WRITE(LPT_TRANSCONF, val);
1711         if (intel_wait_for_register(&dev_priv->uncore,
1712                                     LPT_TRANSCONF,
1713                                     TRANS_STATE_ENABLE,
1714                                     TRANS_STATE_ENABLE,
1715                                     100))
1716                 DRM_ERROR("Failed to enable PCH transcoder\n");
1717 }
1718
1719 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1720                                             enum pipe pipe)
1721 {
1722         i915_reg_t reg;
1723         u32 val;
1724
1725         /* FDI relies on the transcoder */
1726         assert_fdi_tx_disabled(dev_priv, pipe);
1727         assert_fdi_rx_disabled(dev_priv, pipe);
1728
1729         /* Ports must be off as well */
1730         assert_pch_ports_disabled(dev_priv, pipe);
1731
1732         reg = PCH_TRANSCONF(pipe);
1733         val = I915_READ(reg);
1734         val &= ~TRANS_ENABLE;
1735         I915_WRITE(reg, val);
1736         /* wait for PCH transcoder off, transcoder state */
1737         if (intel_wait_for_register(&dev_priv->uncore,
1738                                     reg, TRANS_STATE_ENABLE, 0,
1739                                     50))
1740                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1741
1742         if (HAS_PCH_CPT(dev_priv)) {
1743                 /* Workaround: Clear the timing override chicken bit again. */
1744                 reg = TRANS_CHICKEN2(pipe);
1745                 val = I915_READ(reg);
1746                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1747                 I915_WRITE(reg, val);
1748         }
1749 }
1750
1751 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1752 {
1753         u32 val;
1754
1755         val = I915_READ(LPT_TRANSCONF);
1756         val &= ~TRANS_ENABLE;
1757         I915_WRITE(LPT_TRANSCONF, val);
1758         /* wait for PCH transcoder off, transcoder state */
1759         if (intel_wait_for_register(&dev_priv->uncore,
1760                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1761                                     50))
1762                 DRM_ERROR("Failed to disable PCH transcoder\n");
1763
1764         /* Workaround: clear timing override bit. */
1765         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1766         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1767         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1768 }
1769
1770 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1771 {
1772         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1773
1774         if (HAS_PCH_LPT(dev_priv))
1775                 return PIPE_A;
1776         else
1777                 return crtc->pipe;
1778 }
1779
1780 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1781 {
1782         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1783
1784         /*
1785          * On i965gm the hardware frame counter reads
1786          * zero when the TV encoder is enabled :(
1787          */
1788         if (IS_I965GM(dev_priv) &&
1789             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1790                 return 0;
1791
1792         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1793                 return 0xffffffff; /* full 32 bit counter */
1794         else if (INTEL_GEN(dev_priv) >= 3)
1795                 return 0xffffff; /* only 24 bits of frame count */
1796         else
1797                 return 0; /* Gen2 doesn't have a hardware frame counter */
1798 }
1799
1800 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1801 {
1802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1803
1804         drm_crtc_set_max_vblank_count(&crtc->base,
1805                                       intel_crtc_max_vblank_count(crtc_state));
1806         drm_crtc_vblank_on(&crtc->base);
1807 }
1808
1809 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1810 {
1811         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1813         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1814         enum pipe pipe = crtc->pipe;
1815         i915_reg_t reg;
1816         u32 val;
1817
1818         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1819
1820         assert_planes_disabled(crtc);
1821
1822         /*
1823          * A pipe without a PLL won't actually be able to drive bits from
1824          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1825          * need the check.
1826          */
1827         if (HAS_GMCH(dev_priv)) {
1828                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1829                         assert_dsi_pll_enabled(dev_priv);
1830                 else
1831                         assert_pll_enabled(dev_priv, pipe);
1832         } else {
1833                 if (new_crtc_state->has_pch_encoder) {
1834                         /* if driving the PCH, we need FDI enabled */
1835                         assert_fdi_rx_pll_enabled(dev_priv,
1836                                                   intel_crtc_pch_transcoder(crtc));
1837                         assert_fdi_tx_pll_enabled(dev_priv,
1838                                                   (enum pipe) cpu_transcoder);
1839                 }
1840                 /* FIXME: assert CPU port conditions for SNB+ */
1841         }
1842
1843         trace_intel_pipe_enable(crtc);
1844
1845         reg = PIPECONF(cpu_transcoder);
1846         val = I915_READ(reg);
1847         if (val & PIPECONF_ENABLE) {
1848                 /* we keep both pipes enabled on 830 */
1849                 WARN_ON(!IS_I830(dev_priv));
1850                 return;
1851         }
1852
1853         I915_WRITE(reg, val | PIPECONF_ENABLE);
1854         POSTING_READ(reg);
1855
1856         /*
1857          * Until the pipe starts PIPEDSL reads will return a stale value,
1858          * which causes an apparent vblank timestamp jump when PIPEDSL
1859          * resets to its proper value. That also messes up the frame count
1860          * when it's derived from the timestamps. So let's wait for the
1861          * pipe to start properly before we call drm_crtc_vblank_on()
1862          */
1863         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1864                 intel_wait_for_pipe_scanline_moving(crtc);
1865 }
1866
1867 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1868 {
1869         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1870         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1871         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1872         enum pipe pipe = crtc->pipe;
1873         i915_reg_t reg;
1874         u32 val;
1875
1876         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1877
1878         /*
1879          * Make sure planes won't keep trying to pump pixels to us,
1880          * or we might hang the display.
1881          */
1882         assert_planes_disabled(crtc);
1883
1884         trace_intel_pipe_disable(crtc);
1885
1886         reg = PIPECONF(cpu_transcoder);
1887         val = I915_READ(reg);
1888         if ((val & PIPECONF_ENABLE) == 0)
1889                 return;
1890
1891         /*
1892          * Double wide has implications for planes
1893          * so best keep it disabled when not needed.
1894          */
1895         if (old_crtc_state->double_wide)
1896                 val &= ~PIPECONF_DOUBLE_WIDE;
1897
1898         /* Don't disable pipe or pipe PLLs if needed */
1899         if (!IS_I830(dev_priv))
1900                 val &= ~PIPECONF_ENABLE;
1901
1902         I915_WRITE(reg, val);
1903         if ((val & PIPECONF_ENABLE) == 0)
1904                 intel_wait_for_pipe_off(old_crtc_state);
1905 }
1906
1907 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1908 {
1909         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1910 }
1911
1912 static unsigned int
1913 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1914 {
1915         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1916         unsigned int cpp = fb->format->cpp[color_plane];
1917
1918         switch (fb->modifier) {
1919         case DRM_FORMAT_MOD_LINEAR:
1920                 return intel_tile_size(dev_priv);
1921         case I915_FORMAT_MOD_X_TILED:
1922                 if (IS_GEN(dev_priv, 2))
1923                         return 128;
1924                 else
1925                         return 512;
1926         case I915_FORMAT_MOD_Y_TILED_CCS:
1927                 if (color_plane == 1)
1928                         return 128;
1929                 /* fall through */
1930         case I915_FORMAT_MOD_Y_TILED:
1931                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1932                         return 128;
1933                 else
1934                         return 512;
1935         case I915_FORMAT_MOD_Yf_TILED_CCS:
1936                 if (color_plane == 1)
1937                         return 128;
1938                 /* fall through */
1939         case I915_FORMAT_MOD_Yf_TILED:
1940                 switch (cpp) {
1941                 case 1:
1942                         return 64;
1943                 case 2:
1944                 case 4:
1945                         return 128;
1946                 case 8:
1947                 case 16:
1948                         return 256;
1949                 default:
1950                         MISSING_CASE(cpp);
1951                         return cpp;
1952                 }
1953                 break;
1954         default:
1955                 MISSING_CASE(fb->modifier);
1956                 return cpp;
1957         }
1958 }
1959
1960 static unsigned int
1961 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1962 {
1963         return intel_tile_size(to_i915(fb->dev)) /
1964                 intel_tile_width_bytes(fb, color_plane);
1965 }
1966
1967 /* Return the tile dimensions in pixel units */
1968 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1969                             unsigned int *tile_width,
1970                             unsigned int *tile_height)
1971 {
1972         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1973         unsigned int cpp = fb->format->cpp[color_plane];
1974
1975         *tile_width = tile_width_bytes / cpp;
1976         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1977 }
1978
1979 unsigned int
1980 intel_fb_align_height(const struct drm_framebuffer *fb,
1981                       int color_plane, unsigned int height)
1982 {
1983         unsigned int tile_height = intel_tile_height(fb, color_plane);
1984
1985         return ALIGN(height, tile_height);
1986 }
1987
1988 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1989 {
1990         unsigned int size = 0;
1991         int i;
1992
1993         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1994                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1995
1996         return size;
1997 }
1998
1999 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2000 {
2001         unsigned int size = 0;
2002         int i;
2003
2004         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2005                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2006
2007         return size;
2008 }
2009
2010 static void
2011 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2012                         const struct drm_framebuffer *fb,
2013                         unsigned int rotation)
2014 {
2015         view->type = I915_GGTT_VIEW_NORMAL;
2016         if (drm_rotation_90_or_270(rotation)) {
2017                 view->type = I915_GGTT_VIEW_ROTATED;
2018                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2019         }
2020 }
2021
2022 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2023 {
2024         if (IS_I830(dev_priv))
2025                 return 16 * 1024;
2026         else if (IS_I85X(dev_priv))
2027                 return 256;
2028         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2029                 return 32;
2030         else
2031                 return 4 * 1024;
2032 }
2033
2034 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2035 {
2036         if (INTEL_GEN(dev_priv) >= 9)
2037                 return 256 * 1024;
2038         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2039                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2040                 return 128 * 1024;
2041         else if (INTEL_GEN(dev_priv) >= 4)
2042                 return 4 * 1024;
2043         else
2044                 return 0;
2045 }
2046
2047 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2048                                          int color_plane)
2049 {
2050         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2051
2052         /* AUX_DIST needs only 4K alignment */
2053         if (color_plane == 1)
2054                 return 4096;
2055
2056         switch (fb->modifier) {
2057         case DRM_FORMAT_MOD_LINEAR:
2058                 return intel_linear_alignment(dev_priv);
2059         case I915_FORMAT_MOD_X_TILED:
2060                 if (INTEL_GEN(dev_priv) >= 9)
2061                         return 256 * 1024;
2062                 return 0;
2063         case I915_FORMAT_MOD_Y_TILED_CCS:
2064         case I915_FORMAT_MOD_Yf_TILED_CCS:
2065         case I915_FORMAT_MOD_Y_TILED:
2066         case I915_FORMAT_MOD_Yf_TILED:
2067                 return 1 * 1024 * 1024;
2068         default:
2069                 MISSING_CASE(fb->modifier);
2070                 return 0;
2071         }
2072 }
2073
2074 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2075 {
2076         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2077         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2078
2079         return INTEL_GEN(dev_priv) < 4 ||
2080                 (plane->has_fbc &&
2081                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2082 }
2083
2084 struct i915_vma *
2085 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2086                            const struct i915_ggtt_view *view,
2087                            bool uses_fence,
2088                            unsigned long *out_flags)
2089 {
2090         struct drm_device *dev = fb->dev;
2091         struct drm_i915_private *dev_priv = to_i915(dev);
2092         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2093         intel_wakeref_t wakeref;
2094         struct i915_vma *vma;
2095         unsigned int pinctl;
2096         u32 alignment;
2097
2098         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2099
2100         alignment = intel_surf_alignment(fb, 0);
2101
2102         /* Note that the w/a also requires 64 PTE of padding following the
2103          * bo. We currently fill all unused PTE with the shadow page and so
2104          * we should always have valid PTE following the scanout preventing
2105          * the VT-d warning.
2106          */
2107         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2108                 alignment = 256 * 1024;
2109
2110         /*
2111          * Global gtt pte registers are special registers which actually forward
2112          * writes to a chunk of system memory. Which means that there is no risk
2113          * that the register values disappear as soon as we call
2114          * intel_runtime_pm_put(), so it is correct to wrap only the
2115          * pin/unpin/fence and not more.
2116          */
2117         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2118         i915_gem_object_lock(obj);
2119
2120         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2121
2122         pinctl = 0;
2123
2124         /* Valleyview is definitely limited to scanning out the first
2125          * 512MiB. Lets presume this behaviour was inherited from the
2126          * g4x display engine and that all earlier gen are similarly
2127          * limited. Testing suggests that it is a little more
2128          * complicated than this. For example, Cherryview appears quite
2129          * happy to scanout from anywhere within its global aperture.
2130          */
2131         if (HAS_GMCH(dev_priv))
2132                 pinctl |= PIN_MAPPABLE;
2133
2134         vma = i915_gem_object_pin_to_display_plane(obj,
2135                                                    alignment, view, pinctl);
2136         if (IS_ERR(vma))
2137                 goto err;
2138
2139         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2140                 int ret;
2141
2142                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2143                  * fence, whereas 965+ only requires a fence if using
2144                  * framebuffer compression.  For simplicity, we always, when
2145                  * possible, install a fence as the cost is not that onerous.
2146                  *
2147                  * If we fail to fence the tiled scanout, then either the
2148                  * modeset will reject the change (which is highly unlikely as
2149                  * the affected systems, all but one, do not have unmappable
2150                  * space) or we will not be able to enable full powersaving
2151                  * techniques (also likely not to apply due to various limits
2152                  * FBC and the like impose on the size of the buffer, which
2153                  * presumably we violated anyway with this unmappable buffer).
2154                  * Anyway, it is presumably better to stumble onwards with
2155                  * something and try to run the system in a "less than optimal"
2156                  * mode that matches the user configuration.
2157                  */
2158                 ret = i915_vma_pin_fence(vma);
2159                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2160                         i915_gem_object_unpin_from_display_plane(vma);
2161                         vma = ERR_PTR(ret);
2162                         goto err;
2163                 }
2164
2165                 if (ret == 0 && vma->fence)
2166                         *out_flags |= PLANE_HAS_FENCE;
2167         }
2168
2169         i915_vma_get(vma);
2170 err:
2171         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2172
2173         i915_gem_object_unlock(obj);
2174         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2175         return vma;
2176 }
2177
2178 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2179 {
2180         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2181
2182         i915_gem_object_lock(vma->obj);
2183         if (flags & PLANE_HAS_FENCE)
2184                 i915_vma_unpin_fence(vma);
2185         i915_gem_object_unpin_from_display_plane(vma);
2186         i915_gem_object_unlock(vma->obj);
2187
2188         i915_vma_put(vma);
2189 }
2190
2191 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2192                           unsigned int rotation)
2193 {
2194         if (drm_rotation_90_or_270(rotation))
2195                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2196         else
2197                 return fb->pitches[color_plane];
2198 }
2199
2200 /*
2201  * Convert the x/y offsets into a linear offset.
2202  * Only valid with 0/180 degree rotation, which is fine since linear
2203  * offset is only used with linear buffers on pre-hsw and tiled buffers
2204  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2205  */
2206 u32 intel_fb_xy_to_linear(int x, int y,
2207                           const struct intel_plane_state *state,
2208                           int color_plane)
2209 {
2210         const struct drm_framebuffer *fb = state->base.fb;
2211         unsigned int cpp = fb->format->cpp[color_plane];
2212         unsigned int pitch = state->color_plane[color_plane].stride;
2213
2214         return y * pitch + x * cpp;
2215 }
2216
2217 /*
2218  * Add the x/y offsets derived from fb->offsets[] to the user
2219  * specified plane src x/y offsets. The resulting x/y offsets
2220  * specify the start of scanout from the beginning of the gtt mapping.
2221  */
2222 void intel_add_fb_offsets(int *x, int *y,
2223                           const struct intel_plane_state *state,
2224                           int color_plane)
2225
2226 {
2227         *x += state->color_plane[color_plane].x;
2228         *y += state->color_plane[color_plane].y;
2229 }
2230
2231 static u32 intel_adjust_tile_offset(int *x, int *y,
2232                                     unsigned int tile_width,
2233                                     unsigned int tile_height,
2234                                     unsigned int tile_size,
2235                                     unsigned int pitch_tiles,
2236                                     u32 old_offset,
2237                                     u32 new_offset)
2238 {
2239         unsigned int pitch_pixels = pitch_tiles * tile_width;
2240         unsigned int tiles;
2241
2242         WARN_ON(old_offset & (tile_size - 1));
2243         WARN_ON(new_offset & (tile_size - 1));
2244         WARN_ON(new_offset > old_offset);
2245
2246         tiles = (old_offset - new_offset) / tile_size;
2247
2248         *y += tiles / pitch_tiles * tile_height;
2249         *x += tiles % pitch_tiles * tile_width;
2250
2251         /* minimize x in case it got needlessly big */
2252         *y += *x / pitch_pixels * tile_height;
2253         *x %= pitch_pixels;
2254
2255         return new_offset;
2256 }
2257
2258 static bool is_surface_linear(u64 modifier, int color_plane)
2259 {
2260         return modifier == DRM_FORMAT_MOD_LINEAR;
2261 }
2262
2263 static u32 intel_adjust_aligned_offset(int *x, int *y,
2264                                        const struct drm_framebuffer *fb,
2265                                        int color_plane,
2266                                        unsigned int rotation,
2267                                        unsigned int pitch,
2268                                        u32 old_offset, u32 new_offset)
2269 {
2270         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2271         unsigned int cpp = fb->format->cpp[color_plane];
2272
2273         WARN_ON(new_offset > old_offset);
2274
2275         if (!is_surface_linear(fb->modifier, color_plane)) {
2276                 unsigned int tile_size, tile_width, tile_height;
2277                 unsigned int pitch_tiles;
2278
2279                 tile_size = intel_tile_size(dev_priv);
2280                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2281
2282                 if (drm_rotation_90_or_270(rotation)) {
2283                         pitch_tiles = pitch / tile_height;
2284                         swap(tile_width, tile_height);
2285                 } else {
2286                         pitch_tiles = pitch / (tile_width * cpp);
2287                 }
2288
2289                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2290                                          tile_size, pitch_tiles,
2291                                          old_offset, new_offset);
2292         } else {
2293                 old_offset += *y * pitch + *x * cpp;
2294
2295                 *y = (old_offset - new_offset) / pitch;
2296                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2297         }
2298
2299         return new_offset;
2300 }
2301
2302 /*
2303  * Adjust the tile offset by moving the difference into
2304  * the x/y offsets.
2305  */
2306 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2307                                              const struct intel_plane_state *state,
2308                                              int color_plane,
2309                                              u32 old_offset, u32 new_offset)
2310 {
2311         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2312                                            state->base.rotation,
2313                                            state->color_plane[color_plane].stride,
2314                                            old_offset, new_offset);
2315 }
2316
2317 /*
2318  * Computes the aligned offset to the base tile and adjusts
2319  * x, y. bytes per pixel is assumed to be a power-of-two.
2320  *
2321  * In the 90/270 rotated case, x and y are assumed
2322  * to be already rotated to match the rotated GTT view, and
2323  * pitch is the tile_height aligned framebuffer height.
2324  *
2325  * This function is used when computing the derived information
2326  * under intel_framebuffer, so using any of that information
2327  * here is not allowed. Anything under drm_framebuffer can be
2328  * used. This is why the user has to pass in the pitch since it
2329  * is specified in the rotated orientation.
2330  */
2331 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2332                                         int *x, int *y,
2333                                         const struct drm_framebuffer *fb,
2334                                         int color_plane,
2335                                         unsigned int pitch,
2336                                         unsigned int rotation,
2337                                         u32 alignment)
2338 {
2339         unsigned int cpp = fb->format->cpp[color_plane];
2340         u32 offset, offset_aligned;
2341
2342         if (alignment)
2343                 alignment--;
2344
2345         if (!is_surface_linear(fb->modifier, color_plane)) {
2346                 unsigned int tile_size, tile_width, tile_height;
2347                 unsigned int tile_rows, tiles, pitch_tiles;
2348
2349                 tile_size = intel_tile_size(dev_priv);
2350                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2351
2352                 if (drm_rotation_90_or_270(rotation)) {
2353                         pitch_tiles = pitch / tile_height;
2354                         swap(tile_width, tile_height);
2355                 } else {
2356                         pitch_tiles = pitch / (tile_width * cpp);
2357                 }
2358
2359                 tile_rows = *y / tile_height;
2360                 *y %= tile_height;
2361
2362                 tiles = *x / tile_width;
2363                 *x %= tile_width;
2364
2365                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2366                 offset_aligned = offset & ~alignment;
2367
2368                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2369                                          tile_size, pitch_tiles,
2370                                          offset, offset_aligned);
2371         } else {
2372                 offset = *y * pitch + *x * cpp;
2373                 offset_aligned = offset & ~alignment;
2374
2375                 *y = (offset & alignment) / pitch;
2376                 *x = ((offset & alignment) - *y * pitch) / cpp;
2377         }
2378
2379         return offset_aligned;
2380 }
2381
2382 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2383                                               const struct intel_plane_state *state,
2384                                               int color_plane)
2385 {
2386         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2387         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2388         const struct drm_framebuffer *fb = state->base.fb;
2389         unsigned int rotation = state->base.rotation;
2390         int pitch = state->color_plane[color_plane].stride;
2391         u32 alignment;
2392
2393         if (intel_plane->id == PLANE_CURSOR)
2394                 alignment = intel_cursor_alignment(dev_priv);
2395         else
2396                 alignment = intel_surf_alignment(fb, color_plane);
2397
2398         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2399                                             pitch, rotation, alignment);
2400 }
2401
2402 /* Convert the fb->offset[] into x/y offsets */
2403 static int intel_fb_offset_to_xy(int *x, int *y,
2404                                  const struct drm_framebuffer *fb,
2405                                  int color_plane)
2406 {
2407         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2408         unsigned int height;
2409
2410         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2411             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2412                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2413                               fb->offsets[color_plane], color_plane);
2414                 return -EINVAL;
2415         }
2416
2417         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2418         height = ALIGN(height, intel_tile_height(fb, color_plane));
2419
2420         /* Catch potential overflows early */
2421         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2422                             fb->offsets[color_plane])) {
2423                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2424                               fb->offsets[color_plane], fb->pitches[color_plane],
2425                               color_plane);
2426                 return -ERANGE;
2427         }
2428
2429         *x = 0;
2430         *y = 0;
2431
2432         intel_adjust_aligned_offset(x, y,
2433                                     fb, color_plane, DRM_MODE_ROTATE_0,
2434                                     fb->pitches[color_plane],
2435                                     fb->offsets[color_plane], 0);
2436
2437         return 0;
2438 }
2439
2440 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2441 {
2442         switch (fb_modifier) {
2443         case I915_FORMAT_MOD_X_TILED:
2444                 return I915_TILING_X;
2445         case I915_FORMAT_MOD_Y_TILED:
2446         case I915_FORMAT_MOD_Y_TILED_CCS:
2447                 return I915_TILING_Y;
2448         default:
2449                 return I915_TILING_NONE;
2450         }
2451 }
2452
2453 /*
2454  * From the Sky Lake PRM:
2455  * "The Color Control Surface (CCS) contains the compression status of
2456  *  the cache-line pairs. The compression state of the cache-line pair
2457  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2458  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2459  *  cache-line-pairs. CCS is always Y tiled."
2460  *
2461  * Since cache line pairs refers to horizontally adjacent cache lines,
2462  * each cache line in the CCS corresponds to an area of 32x16 cache
2463  * lines on the main surface. Since each pixel is 4 bytes, this gives
2464  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2465  * main surface.
2466  */
2467 static const struct drm_format_info ccs_formats[] = {
2468         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2469           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2470         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2471           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2472         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2473           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2474         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2475           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2476 };
2477
2478 static const struct drm_format_info *
2479 lookup_format_info(const struct drm_format_info formats[],
2480                    int num_formats, u32 format)
2481 {
2482         int i;
2483
2484         for (i = 0; i < num_formats; i++) {
2485                 if (formats[i].format == format)
2486                         return &formats[i];
2487         }
2488
2489         return NULL;
2490 }
2491
2492 static const struct drm_format_info *
2493 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2494 {
2495         switch (cmd->modifier[0]) {
2496         case I915_FORMAT_MOD_Y_TILED_CCS:
2497         case I915_FORMAT_MOD_Yf_TILED_CCS:
2498                 return lookup_format_info(ccs_formats,
2499                                           ARRAY_SIZE(ccs_formats),
2500                                           cmd->pixel_format);
2501         default:
2502                 return NULL;
2503         }
2504 }
2505
2506 bool is_ccs_modifier(u64 modifier)
2507 {
2508         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2509                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2510 }
2511
2512 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2513                               u32 pixel_format, u64 modifier)
2514 {
2515         struct intel_crtc *crtc;
2516         struct intel_plane *plane;
2517
2518         /*
2519          * We assume the primary plane for pipe A has
2520          * the highest stride limits of them all.
2521          */
2522         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2523         plane = to_intel_plane(crtc->base.primary);
2524
2525         return plane->max_stride(plane, pixel_format, modifier,
2526                                  DRM_MODE_ROTATE_0);
2527 }
2528
2529 static
2530 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2531                         u32 pixel_format, u64 modifier)
2532 {
2533         /*
2534          * Arbitrary limit for gen4+ chosen to match the
2535          * render engine max stride.
2536          *
2537          * The new CCS hash mode makes remapping impossible
2538          */
2539         if (!is_ccs_modifier(modifier)) {
2540                 if (INTEL_GEN(dev_priv) >= 7)
2541                         return 256*1024;
2542                 else if (INTEL_GEN(dev_priv) >= 4)
2543                         return 128*1024;
2544         }
2545
2546         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2547 }
2548
2549 static u32
2550 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2551 {
2552         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2553
2554         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2555                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2556                                                            fb->format->format,
2557                                                            fb->modifier);
2558
2559                 /*
2560                  * To make remapping with linear generally feasible
2561                  * we need the stride to be page aligned.
2562                  */
2563                 if (fb->pitches[color_plane] > max_stride)
2564                         return intel_tile_size(dev_priv);
2565                 else
2566                         return 64;
2567         } else {
2568                 return intel_tile_width_bytes(fb, color_plane);
2569         }
2570 }
2571
2572 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2573 {
2574         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2575         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2576         const struct drm_framebuffer *fb = plane_state->base.fb;
2577         int i;
2578
2579         /* We don't want to deal with remapping with cursors */
2580         if (plane->id == PLANE_CURSOR)
2581                 return false;
2582
2583         /*
2584          * The display engine limits already match/exceed the
2585          * render engine limits, so not much point in remapping.
2586          * Would also need to deal with the fence POT alignment
2587          * and gen2 2KiB GTT tile size.
2588          */
2589         if (INTEL_GEN(dev_priv) < 4)
2590                 return false;
2591
2592         /*
2593          * The new CCS hash mode isn't compatible with remapping as
2594          * the virtual address of the pages affects the compressed data.
2595          */
2596         if (is_ccs_modifier(fb->modifier))
2597                 return false;
2598
2599         /* Linear needs a page aligned stride for remapping */
2600         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2601                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2602
2603                 for (i = 0; i < fb->format->num_planes; i++) {
2604                         if (fb->pitches[i] & alignment)
2605                                 return false;
2606                 }
2607         }
2608
2609         return true;
2610 }
2611
2612 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2613 {
2614         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2615         const struct drm_framebuffer *fb = plane_state->base.fb;
2616         unsigned int rotation = plane_state->base.rotation;
2617         u32 stride, max_stride;
2618
2619         /*
2620          * No remapping for invisible planes since we don't have
2621          * an actual source viewport to remap.
2622          */
2623         if (!plane_state->base.visible)
2624                 return false;
2625
2626         if (!intel_plane_can_remap(plane_state))
2627                 return false;
2628
2629         /*
2630          * FIXME: aux plane limits on gen9+ are
2631          * unclear in Bspec, for now no checking.
2632          */
2633         stride = intel_fb_pitch(fb, 0, rotation);
2634         max_stride = plane->max_stride(plane, fb->format->format,
2635                                        fb->modifier, rotation);
2636
2637         return stride > max_stride;
2638 }
2639
2640 static int
2641 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2642                    struct drm_framebuffer *fb)
2643 {
2644         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2645         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2646         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2647         u32 gtt_offset_rotated = 0;
2648         unsigned int max_size = 0;
2649         int i, num_planes = fb->format->num_planes;
2650         unsigned int tile_size = intel_tile_size(dev_priv);
2651
2652         for (i = 0; i < num_planes; i++) {
2653                 unsigned int width, height;
2654                 unsigned int cpp, size;
2655                 u32 offset;
2656                 int x, y;
2657                 int ret;
2658
2659                 cpp = fb->format->cpp[i];
2660                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2661                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2662
2663                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2664                 if (ret) {
2665                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2666                                       i, fb->offsets[i]);
2667                         return ret;
2668                 }
2669
2670                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2671                         int hsub = fb->format->hsub;
2672                         int vsub = fb->format->vsub;
2673                         int tile_width, tile_height;
2674                         int main_x, main_y;
2675                         int ccs_x, ccs_y;
2676
2677                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2678                         tile_width *= hsub;
2679                         tile_height *= vsub;
2680
2681                         ccs_x = (x * hsub) % tile_width;
2682                         ccs_y = (y * vsub) % tile_height;
2683                         main_x = intel_fb->normal[0].x % tile_width;
2684                         main_y = intel_fb->normal[0].y % tile_height;
2685
2686                         /*
2687                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2688                          * x/y offsets must match between CCS and the main surface.
2689                          */
2690                         if (main_x != ccs_x || main_y != ccs_y) {
2691                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2692                                               main_x, main_y,
2693                                               ccs_x, ccs_y,
2694                                               intel_fb->normal[0].x,
2695                                               intel_fb->normal[0].y,
2696                                               x, y);
2697                                 return -EINVAL;
2698                         }
2699                 }
2700
2701                 /*
2702                  * The fence (if used) is aligned to the start of the object
2703                  * so having the framebuffer wrap around across the edge of the
2704                  * fenced region doesn't really work. We have no API to configure
2705                  * the fence start offset within the object (nor could we probably
2706                  * on gen2/3). So it's just easier if we just require that the
2707                  * fb layout agrees with the fence layout. We already check that the
2708                  * fb stride matches the fence stride elsewhere.
2709                  */
2710                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2711                     (x + width) * cpp > fb->pitches[i]) {
2712                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2713                                       i, fb->offsets[i]);
2714                         return -EINVAL;
2715                 }
2716
2717                 /*
2718                  * First pixel of the framebuffer from
2719                  * the start of the normal gtt mapping.
2720                  */
2721                 intel_fb->normal[i].x = x;
2722                 intel_fb->normal[i].y = y;
2723
2724                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2725                                                       fb->pitches[i],
2726                                                       DRM_MODE_ROTATE_0,
2727                                                       tile_size);
2728                 offset /= tile_size;
2729
2730                 if (!is_surface_linear(fb->modifier, i)) {
2731                         unsigned int tile_width, tile_height;
2732                         unsigned int pitch_tiles;
2733                         struct drm_rect r;
2734
2735                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2736
2737                         rot_info->plane[i].offset = offset;
2738                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2739                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2740                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2741
2742                         intel_fb->rotated[i].pitch =
2743                                 rot_info->plane[i].height * tile_height;
2744
2745                         /* how many tiles does this plane need */
2746                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2747                         /*
2748                          * If the plane isn't horizontally tile aligned,
2749                          * we need one more tile.
2750                          */
2751                         if (x != 0)
2752                                 size++;
2753
2754                         /* rotate the x/y offsets to match the GTT view */
2755                         r.x1 = x;
2756                         r.y1 = y;
2757                         r.x2 = x + width;
2758                         r.y2 = y + height;
2759                         drm_rect_rotate(&r,
2760                                         rot_info->plane[i].width * tile_width,
2761                                         rot_info->plane[i].height * tile_height,
2762                                         DRM_MODE_ROTATE_270);
2763                         x = r.x1;
2764                         y = r.y1;
2765
2766                         /* rotate the tile dimensions to match the GTT view */
2767                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2768                         swap(tile_width, tile_height);
2769
2770                         /*
2771                          * We only keep the x/y offsets, so push all of the
2772                          * gtt offset into the x/y offsets.
2773                          */
2774                         intel_adjust_tile_offset(&x, &y,
2775                                                  tile_width, tile_height,
2776                                                  tile_size, pitch_tiles,
2777                                                  gtt_offset_rotated * tile_size, 0);
2778
2779                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2780
2781                         /*
2782                          * First pixel of the framebuffer from
2783                          * the start of the rotated gtt mapping.
2784                          */
2785                         intel_fb->rotated[i].x = x;
2786                         intel_fb->rotated[i].y = y;
2787                 } else {
2788                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2789                                             x * cpp, tile_size);
2790                 }
2791
2792                 /* how many tiles in total needed in the bo */
2793                 max_size = max(max_size, offset + size);
2794         }
2795
2796         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2797                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2798                               mul_u32_u32(max_size, tile_size), obj->base.size);
2799                 return -EINVAL;
2800         }
2801
2802         return 0;
2803 }
2804
2805 static void
2806 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2807 {
2808         struct drm_i915_private *dev_priv =
2809                 to_i915(plane_state->base.plane->dev);
2810         struct drm_framebuffer *fb = plane_state->base.fb;
2811         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2812         struct intel_rotation_info *info = &plane_state->view.rotated;
2813         unsigned int rotation = plane_state->base.rotation;
2814         int i, num_planes = fb->format->num_planes;
2815         unsigned int tile_size = intel_tile_size(dev_priv);
2816         unsigned int src_x, src_y;
2817         unsigned int src_w, src_h;
2818         u32 gtt_offset = 0;
2819
2820         memset(&plane_state->view, 0, sizeof(plane_state->view));
2821         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2822                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2823
2824         src_x = plane_state->base.src.x1 >> 16;
2825         src_y = plane_state->base.src.y1 >> 16;
2826         src_w = drm_rect_width(&plane_state->base.src) >> 16;
2827         src_h = drm_rect_height(&plane_state->base.src) >> 16;
2828
2829         WARN_ON(is_ccs_modifier(fb->modifier));
2830
2831         /* Make src coordinates relative to the viewport */
2832         drm_rect_translate(&plane_state->base.src,
2833                            -(src_x << 16), -(src_y << 16));
2834
2835         /* Rotate src coordinates to match rotated GTT view */
2836         if (drm_rotation_90_or_270(rotation))
2837                 drm_rect_rotate(&plane_state->base.src,
2838                                 src_w << 16, src_h << 16,
2839                                 DRM_MODE_ROTATE_270);
2840
2841         for (i = 0; i < num_planes; i++) {
2842                 unsigned int hsub = i ? fb->format->hsub : 1;
2843                 unsigned int vsub = i ? fb->format->vsub : 1;
2844                 unsigned int cpp = fb->format->cpp[i];
2845                 unsigned int tile_width, tile_height;
2846                 unsigned int width, height;
2847                 unsigned int pitch_tiles;
2848                 unsigned int x, y;
2849                 u32 offset;
2850
2851                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2852
2853                 x = src_x / hsub;
2854                 y = src_y / vsub;
2855                 width = src_w / hsub;
2856                 height = src_h / vsub;
2857
2858                 /*
2859                  * First pixel of the src viewport from the
2860                  * start of the normal gtt mapping.
2861                  */
2862                 x += intel_fb->normal[i].x;
2863                 y += intel_fb->normal[i].y;
2864
2865                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2866                                                       fb, i, fb->pitches[i],
2867                                                       DRM_MODE_ROTATE_0, tile_size);
2868                 offset /= tile_size;
2869
2870                 info->plane[i].offset = offset;
2871                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2872                                                      tile_width * cpp);
2873                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2874                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2875
2876                 if (drm_rotation_90_or_270(rotation)) {
2877                         struct drm_rect r;
2878
2879                         /* rotate the x/y offsets to match the GTT view */
2880                         r.x1 = x;
2881                         r.y1 = y;
2882                         r.x2 = x + width;
2883                         r.y2 = y + height;
2884                         drm_rect_rotate(&r,
2885                                         info->plane[i].width * tile_width,
2886                                         info->plane[i].height * tile_height,
2887                                         DRM_MODE_ROTATE_270);
2888                         x = r.x1;
2889                         y = r.y1;
2890
2891                         pitch_tiles = info->plane[i].height;
2892                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2893
2894                         /* rotate the tile dimensions to match the GTT view */
2895                         swap(tile_width, tile_height);
2896                 } else {
2897                         pitch_tiles = info->plane[i].width;
2898                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2899                 }
2900
2901                 /*
2902                  * We only keep the x/y offsets, so push all of the
2903                  * gtt offset into the x/y offsets.
2904                  */
2905                 intel_adjust_tile_offset(&x, &y,
2906                                          tile_width, tile_height,
2907                                          tile_size, pitch_tiles,
2908                                          gtt_offset * tile_size, 0);
2909
2910                 gtt_offset += info->plane[i].width * info->plane[i].height;
2911
2912                 plane_state->color_plane[i].offset = 0;
2913                 plane_state->color_plane[i].x = x;
2914                 plane_state->color_plane[i].y = y;
2915         }
2916 }
2917
2918 static int
2919 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2920 {
2921         const struct intel_framebuffer *fb =
2922                 to_intel_framebuffer(plane_state->base.fb);
2923         unsigned int rotation = plane_state->base.rotation;
2924         int i, num_planes;
2925
2926         if (!fb)
2927                 return 0;
2928
2929         num_planes = fb->base.format->num_planes;
2930
2931         if (intel_plane_needs_remap(plane_state)) {
2932                 intel_plane_remap_gtt(plane_state);
2933
2934                 /*
2935                  * Sometimes even remapping can't overcome
2936                  * the stride limitations :( Can happen with
2937                  * big plane sizes and suitably misaligned
2938                  * offsets.
2939                  */
2940                 return intel_plane_check_stride(plane_state);
2941         }
2942
2943         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2944
2945         for (i = 0; i < num_planes; i++) {
2946                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2947                 plane_state->color_plane[i].offset = 0;
2948
2949                 if (drm_rotation_90_or_270(rotation)) {
2950                         plane_state->color_plane[i].x = fb->rotated[i].x;
2951                         plane_state->color_plane[i].y = fb->rotated[i].y;
2952                 } else {
2953                         plane_state->color_plane[i].x = fb->normal[i].x;
2954                         plane_state->color_plane[i].y = fb->normal[i].y;
2955                 }
2956         }
2957
2958         /* Rotate src coordinates to match rotated GTT view */
2959         if (drm_rotation_90_or_270(rotation))
2960                 drm_rect_rotate(&plane_state->base.src,
2961                                 fb->base.width << 16, fb->base.height << 16,
2962                                 DRM_MODE_ROTATE_270);
2963
2964         return intel_plane_check_stride(plane_state);
2965 }
2966
2967 static int i9xx_format_to_fourcc(int format)
2968 {
2969         switch (format) {
2970         case DISPPLANE_8BPP:
2971                 return DRM_FORMAT_C8;
2972         case DISPPLANE_BGRX555:
2973                 return DRM_FORMAT_XRGB1555;
2974         case DISPPLANE_BGRX565:
2975                 return DRM_FORMAT_RGB565;
2976         default:
2977         case DISPPLANE_BGRX888:
2978                 return DRM_FORMAT_XRGB8888;
2979         case DISPPLANE_RGBX888:
2980                 return DRM_FORMAT_XBGR8888;
2981         case DISPPLANE_BGRX101010:
2982                 return DRM_FORMAT_XRGB2101010;
2983         case DISPPLANE_RGBX101010:
2984                 return DRM_FORMAT_XBGR2101010;
2985         }
2986 }
2987
2988 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2989 {
2990         switch (format) {
2991         case PLANE_CTL_FORMAT_RGB_565:
2992                 return DRM_FORMAT_RGB565;
2993         case PLANE_CTL_FORMAT_NV12:
2994                 return DRM_FORMAT_NV12;
2995         case PLANE_CTL_FORMAT_P010:
2996                 return DRM_FORMAT_P010;
2997         case PLANE_CTL_FORMAT_P012:
2998                 return DRM_FORMAT_P012;
2999         case PLANE_CTL_FORMAT_P016:
3000                 return DRM_FORMAT_P016;
3001         case PLANE_CTL_FORMAT_Y210:
3002                 return DRM_FORMAT_Y210;
3003         case PLANE_CTL_FORMAT_Y212:
3004                 return DRM_FORMAT_Y212;
3005         case PLANE_CTL_FORMAT_Y216:
3006                 return DRM_FORMAT_Y216;
3007         case PLANE_CTL_FORMAT_Y410:
3008                 return DRM_FORMAT_XVYU2101010;
3009         case PLANE_CTL_FORMAT_Y412:
3010                 return DRM_FORMAT_XVYU12_16161616;
3011         case PLANE_CTL_FORMAT_Y416:
3012                 return DRM_FORMAT_XVYU16161616;
3013         default:
3014         case PLANE_CTL_FORMAT_XRGB_8888:
3015                 if (rgb_order) {
3016                         if (alpha)
3017                                 return DRM_FORMAT_ABGR8888;
3018                         else
3019                                 return DRM_FORMAT_XBGR8888;
3020                 } else {
3021                         if (alpha)
3022                                 return DRM_FORMAT_ARGB8888;
3023                         else
3024                                 return DRM_FORMAT_XRGB8888;
3025                 }
3026         case PLANE_CTL_FORMAT_XRGB_2101010:
3027                 if (rgb_order)
3028                         return DRM_FORMAT_XBGR2101010;
3029                 else
3030                         return DRM_FORMAT_XRGB2101010;
3031         case PLANE_CTL_FORMAT_XRGB_16161616F:
3032                 if (rgb_order) {
3033                         if (alpha)
3034                                 return DRM_FORMAT_ABGR16161616F;
3035                         else
3036                                 return DRM_FORMAT_XBGR16161616F;
3037                 } else {
3038                         if (alpha)
3039                                 return DRM_FORMAT_ARGB16161616F;
3040                         else
3041                                 return DRM_FORMAT_XRGB16161616F;
3042                 }
3043         }
3044 }
3045
3046 static bool
3047 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3048                               struct intel_initial_plane_config *plane_config)
3049 {
3050         struct drm_device *dev = crtc->base.dev;
3051         struct drm_i915_private *dev_priv = to_i915(dev);
3052         struct drm_i915_gem_object *obj = NULL;
3053         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3054         struct drm_framebuffer *fb = &plane_config->fb->base;
3055         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3056         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3057                                     PAGE_SIZE);
3058
3059         size_aligned -= base_aligned;
3060
3061         if (plane_config->size == 0)
3062                 return false;
3063
3064         /* If the FB is too big, just don't use it since fbdev is not very
3065          * important and we should probably use that space with FBC or other
3066          * features. */
3067         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3068                 return false;
3069
3070         switch (fb->modifier) {
3071         case DRM_FORMAT_MOD_LINEAR:
3072         case I915_FORMAT_MOD_X_TILED:
3073         case I915_FORMAT_MOD_Y_TILED:
3074                 break;
3075         default:
3076                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3077                                  fb->modifier);
3078                 return false;
3079         }
3080
3081         mutex_lock(&dev->struct_mutex);
3082         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3083                                                              base_aligned,
3084                                                              base_aligned,
3085                                                              size_aligned);
3086         mutex_unlock(&dev->struct_mutex);
3087         if (!obj)
3088                 return false;
3089
3090         switch (plane_config->tiling) {
3091         case I915_TILING_NONE:
3092                 break;
3093         case I915_TILING_X:
3094         case I915_TILING_Y:
3095                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3096                 break;
3097         default:
3098                 MISSING_CASE(plane_config->tiling);
3099                 return false;
3100         }
3101
3102         mode_cmd.pixel_format = fb->format->format;
3103         mode_cmd.width = fb->width;
3104         mode_cmd.height = fb->height;
3105         mode_cmd.pitches[0] = fb->pitches[0];
3106         mode_cmd.modifier[0] = fb->modifier;
3107         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3108
3109         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3110                 DRM_DEBUG_KMS("intel fb init failed\n");
3111                 goto out_unref_obj;
3112         }
3113
3114
3115         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3116         return true;
3117
3118 out_unref_obj:
3119         i915_gem_object_put(obj);
3120         return false;
3121 }
3122
3123 static void
3124 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3125                         struct intel_plane_state *plane_state,
3126                         bool visible)
3127 {
3128         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3129
3130         plane_state->base.visible = visible;
3131
3132         if (visible)
3133                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
3134         else
3135                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
3136 }
3137
3138 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3139 {
3140         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3141         struct drm_plane *plane;
3142
3143         /*
3144          * Active_planes aliases if multiple "primary" or cursor planes
3145          * have been used on the same (or wrong) pipe. plane_mask uses
3146          * unique ids, hence we can use that to reconstruct active_planes.
3147          */
3148         crtc_state->active_planes = 0;
3149
3150         drm_for_each_plane_mask(plane, &dev_priv->drm,
3151                                 crtc_state->base.plane_mask)
3152                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3153 }
3154
3155 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3156                                          struct intel_plane *plane)
3157 {
3158         struct intel_crtc_state *crtc_state =
3159                 to_intel_crtc_state(crtc->base.state);
3160         struct intel_plane_state *plane_state =
3161                 to_intel_plane_state(plane->base.state);
3162
3163         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3164                       plane->base.base.id, plane->base.name,
3165                       crtc->base.base.id, crtc->base.name);
3166
3167         intel_set_plane_visible(crtc_state, plane_state, false);
3168         fixup_active_planes(crtc_state);
3169         crtc_state->data_rate[plane->id] = 0;
3170
3171         if (plane->id == PLANE_PRIMARY)
3172                 intel_pre_disable_primary_noatomic(&crtc->base);
3173
3174         intel_disable_plane(plane, crtc_state);
3175 }
3176
3177 static void
3178 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3179                              struct intel_initial_plane_config *plane_config)
3180 {
3181         struct drm_device *dev = intel_crtc->base.dev;
3182         struct drm_i915_private *dev_priv = to_i915(dev);
3183         struct drm_crtc *c;
3184         struct drm_i915_gem_object *obj;
3185         struct drm_plane *primary = intel_crtc->base.primary;
3186         struct drm_plane_state *plane_state = primary->state;
3187         struct intel_plane *intel_plane = to_intel_plane(primary);
3188         struct intel_plane_state *intel_state =
3189                 to_intel_plane_state(plane_state);
3190         struct drm_framebuffer *fb;
3191
3192         if (!plane_config->fb)
3193                 return;
3194
3195         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3196                 fb = &plane_config->fb->base;
3197                 goto valid_fb;
3198         }
3199
3200         kfree(plane_config->fb);
3201
3202         /*
3203          * Failed to alloc the obj, check to see if we should share
3204          * an fb with another CRTC instead
3205          */
3206         for_each_crtc(dev, c) {
3207                 struct intel_plane_state *state;
3208
3209                 if (c == &intel_crtc->base)
3210                         continue;
3211
3212                 if (!to_intel_crtc(c)->active)
3213                         continue;
3214
3215                 state = to_intel_plane_state(c->primary->state);
3216                 if (!state->vma)
3217                         continue;
3218
3219                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3220                         fb = state->base.fb;
3221                         drm_framebuffer_get(fb);
3222                         goto valid_fb;
3223                 }
3224         }
3225
3226         /*
3227          * We've failed to reconstruct the BIOS FB.  Current display state
3228          * indicates that the primary plane is visible, but has a NULL FB,
3229          * which will lead to problems later if we don't fix it up.  The
3230          * simplest solution is to just disable the primary plane now and
3231          * pretend the BIOS never had it enabled.
3232          */
3233         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3234
3235         return;
3236
3237 valid_fb:
3238         intel_state->base.rotation = plane_config->rotation;
3239         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3240                                 intel_state->base.rotation);
3241         intel_state->color_plane[0].stride =
3242                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
3243
3244         mutex_lock(&dev->struct_mutex);
3245         intel_state->vma =
3246                 intel_pin_and_fence_fb_obj(fb,
3247                                            &intel_state->view,
3248                                            intel_plane_uses_fence(intel_state),
3249                                            &intel_state->flags);
3250         mutex_unlock(&dev->struct_mutex);
3251         if (IS_ERR(intel_state->vma)) {
3252                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3253                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3254
3255                 intel_state->vma = NULL;
3256                 drm_framebuffer_put(fb);
3257                 return;
3258         }
3259
3260         obj = intel_fb_obj(fb);
3261         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
3262
3263         plane_state->src_x = 0;
3264         plane_state->src_y = 0;
3265         plane_state->src_w = fb->width << 16;
3266         plane_state->src_h = fb->height << 16;
3267
3268         plane_state->crtc_x = 0;
3269         plane_state->crtc_y = 0;
3270         plane_state->crtc_w = fb->width;
3271         plane_state->crtc_h = fb->height;
3272
3273         intel_state->base.src = drm_plane_state_src(plane_state);
3274         intel_state->base.dst = drm_plane_state_dest(plane_state);
3275
3276         if (i915_gem_object_is_tiled(obj))
3277                 dev_priv->preserve_bios_swizzle = true;
3278
3279         plane_state->fb = fb;
3280         plane_state->crtc = &intel_crtc->base;
3281
3282         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3283                   &obj->frontbuffer_bits);
3284 }
3285
3286 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3287                                int color_plane,
3288                                unsigned int rotation)
3289 {
3290         int cpp = fb->format->cpp[color_plane];
3291
3292         switch (fb->modifier) {
3293         case DRM_FORMAT_MOD_LINEAR:
3294         case I915_FORMAT_MOD_X_TILED:
3295                 return 4096;
3296         case I915_FORMAT_MOD_Y_TILED_CCS:
3297         case I915_FORMAT_MOD_Yf_TILED_CCS:
3298                 /* FIXME AUX plane? */
3299         case I915_FORMAT_MOD_Y_TILED:
3300         case I915_FORMAT_MOD_Yf_TILED:
3301                 if (cpp == 8)
3302                         return 2048;
3303                 else
3304                         return 4096;
3305         default:
3306                 MISSING_CASE(fb->modifier);
3307                 return 2048;
3308         }
3309 }
3310
3311 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3312                                int color_plane,
3313                                unsigned int rotation)
3314 {
3315         int cpp = fb->format->cpp[color_plane];
3316
3317         switch (fb->modifier) {
3318         case DRM_FORMAT_MOD_LINEAR:
3319         case I915_FORMAT_MOD_X_TILED:
3320                 if (cpp == 8)
3321                         return 4096;
3322                 else
3323                         return 5120;
3324         case I915_FORMAT_MOD_Y_TILED_CCS:
3325         case I915_FORMAT_MOD_Yf_TILED_CCS:
3326                 /* FIXME AUX plane? */
3327         case I915_FORMAT_MOD_Y_TILED:
3328         case I915_FORMAT_MOD_Yf_TILED:
3329                 if (cpp == 8)
3330                         return 2048;
3331                 else
3332                         return 5120;
3333         default:
3334                 MISSING_CASE(fb->modifier);
3335                 return 2048;
3336         }
3337 }
3338
3339 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3340                                int color_plane,
3341                                unsigned int rotation)
3342 {
3343         return 5120;
3344 }
3345
3346 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3347                                            int main_x, int main_y, u32 main_offset)
3348 {
3349         const struct drm_framebuffer *fb = plane_state->base.fb;
3350         int hsub = fb->format->hsub;
3351         int vsub = fb->format->vsub;
3352         int aux_x = plane_state->color_plane[1].x;
3353         int aux_y = plane_state->color_plane[1].y;
3354         u32 aux_offset = plane_state->color_plane[1].offset;
3355         u32 alignment = intel_surf_alignment(fb, 1);
3356
3357         while (aux_offset >= main_offset && aux_y <= main_y) {
3358                 int x, y;
3359
3360                 if (aux_x == main_x && aux_y == main_y)
3361                         break;
3362
3363                 if (aux_offset == 0)
3364                         break;
3365
3366                 x = aux_x / hsub;
3367                 y = aux_y / vsub;
3368                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
3369                                                                aux_offset, aux_offset - alignment);
3370                 aux_x = x * hsub + aux_x % hsub;
3371                 aux_y = y * vsub + aux_y % vsub;
3372         }
3373
3374         if (aux_x != main_x || aux_y != main_y)
3375                 return false;
3376
3377         plane_state->color_plane[1].offset = aux_offset;
3378         plane_state->color_plane[1].x = aux_x;
3379         plane_state->color_plane[1].y = aux_y;
3380
3381         return true;
3382 }
3383
3384 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3385 {
3386         struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev);
3387         const struct drm_framebuffer *fb = plane_state->base.fb;
3388         unsigned int rotation = plane_state->base.rotation;
3389         int x = plane_state->base.src.x1 >> 16;
3390         int y = plane_state->base.src.y1 >> 16;
3391         int w = drm_rect_width(&plane_state->base.src) >> 16;
3392         int h = drm_rect_height(&plane_state->base.src) >> 16;
3393         int max_width;
3394         int max_height = 4096;
3395         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3396
3397         if (INTEL_GEN(dev_priv) >= 11)
3398                 max_width = icl_max_plane_width(fb, 0, rotation);
3399         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3400                 max_width = glk_max_plane_width(fb, 0, rotation);
3401         else
3402                 max_width = skl_max_plane_width(fb, 0, rotation);
3403
3404         if (w > max_width || h > max_height) {
3405                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3406                               w, h, max_width, max_height);
3407                 return -EINVAL;
3408         }
3409
3410         intel_add_fb_offsets(&x, &y, plane_state, 0);
3411         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3412         alignment = intel_surf_alignment(fb, 0);
3413
3414         /*
3415          * AUX surface offset is specified as the distance from the
3416          * main surface offset, and it must be non-negative. Make
3417          * sure that is what we will get.
3418          */
3419         if (offset > aux_offset)
3420                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3421                                                            offset, aux_offset & ~(alignment - 1));
3422
3423         /*
3424          * When using an X-tiled surface, the plane blows up
3425          * if the x offset + width exceed the stride.
3426          *
3427          * TODO: linear and Y-tiled seem fine, Yf untested,
3428          */
3429         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3430                 int cpp = fb->format->cpp[0];
3431
3432                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3433                         if (offset == 0) {
3434                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3435                                 return -EINVAL;
3436                         }
3437
3438                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3439                                                                    offset, offset - alignment);
3440                 }
3441         }
3442
3443         /*
3444          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3445          * they match with the main surface x/y offsets.
3446          */
3447         if (is_ccs_modifier(fb->modifier)) {
3448                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3449                         if (offset == 0)
3450                                 break;
3451
3452                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3453                                                                    offset, offset - alignment);
3454                 }
3455
3456                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3457                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3458                         return -EINVAL;
3459                 }
3460         }
3461
3462         plane_state->color_plane[0].offset = offset;
3463         plane_state->color_plane[0].x = x;
3464         plane_state->color_plane[0].y = y;
3465
3466         /*
3467          * Put the final coordinates back so that the src
3468          * coordinate checks will see the right values.
3469          */
3470         drm_rect_translate(&plane_state->base.src,
3471                            (x << 16) - plane_state->base.src.x1,
3472                            (y << 16) - plane_state->base.src.y1);
3473
3474         return 0;
3475 }
3476
3477 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3478 {
3479         const struct drm_framebuffer *fb = plane_state->base.fb;
3480         unsigned int rotation = plane_state->base.rotation;
3481         int max_width = skl_max_plane_width(fb, 1, rotation);
3482         int max_height = 4096;
3483         int x = plane_state->base.src.x1 >> 17;
3484         int y = plane_state->base.src.y1 >> 17;
3485         int w = drm_rect_width(&plane_state->base.src) >> 17;
3486         int h = drm_rect_height(&plane_state->base.src) >> 17;
3487         u32 offset;
3488
3489         intel_add_fb_offsets(&x, &y, plane_state, 1);
3490         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3491
3492         /* FIXME not quite sure how/if these apply to the chroma plane */
3493         if (w > max_width || h > max_height) {
3494                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3495                               w, h, max_width, max_height);
3496                 return -EINVAL;
3497         }
3498
3499         plane_state->color_plane[1].offset = offset;
3500         plane_state->color_plane[1].x = x;
3501         plane_state->color_plane[1].y = y;
3502
3503         return 0;
3504 }
3505
3506 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3507 {
3508         const struct drm_framebuffer *fb = plane_state->base.fb;
3509         int src_x = plane_state->base.src.x1 >> 16;
3510         int src_y = plane_state->base.src.y1 >> 16;
3511         int hsub = fb->format->hsub;
3512         int vsub = fb->format->vsub;
3513         int x = src_x / hsub;
3514         int y = src_y / vsub;
3515         u32 offset;
3516
3517         intel_add_fb_offsets(&x, &y, plane_state, 1);
3518         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3519
3520         plane_state->color_plane[1].offset = offset;
3521         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3522         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3523
3524         return 0;
3525 }
3526
3527 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3528 {
3529         const struct drm_framebuffer *fb = plane_state->base.fb;
3530         int ret;
3531
3532         ret = intel_plane_compute_gtt(plane_state);
3533         if (ret)
3534                 return ret;
3535
3536         if (!plane_state->base.visible)
3537                 return 0;
3538
3539         /*
3540          * Handle the AUX surface first since
3541          * the main surface setup depends on it.
3542          */
3543         if (is_planar_yuv_format(fb->format->format)) {
3544                 ret = skl_check_nv12_aux_surface(plane_state);
3545                 if (ret)
3546                         return ret;
3547         } else if (is_ccs_modifier(fb->modifier)) {
3548                 ret = skl_check_ccs_aux_surface(plane_state);
3549                 if (ret)
3550                         return ret;
3551         } else {
3552                 plane_state->color_plane[1].offset = ~0xfff;
3553                 plane_state->color_plane[1].x = 0;
3554                 plane_state->color_plane[1].y = 0;
3555         }
3556
3557         ret = skl_check_main_surface(plane_state);
3558         if (ret)
3559                 return ret;
3560
3561         return 0;
3562 }
3563
3564 unsigned int
3565 i9xx_plane_max_stride(struct intel_plane *plane,
3566                       u32 pixel_format, u64 modifier,
3567                       unsigned int rotation)
3568 {
3569         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3570
3571         if (!HAS_GMCH(dev_priv)) {
3572                 return 32*1024;
3573         } else if (INTEL_GEN(dev_priv) >= 4) {
3574                 if (modifier == I915_FORMAT_MOD_X_TILED)
3575                         return 16*1024;
3576                 else
3577                         return 32*1024;
3578         } else if (INTEL_GEN(dev_priv) >= 3) {
3579                 if (modifier == I915_FORMAT_MOD_X_TILED)
3580                         return 8*1024;
3581                 else
3582                         return 16*1024;
3583         } else {
3584                 if (plane->i9xx_plane == PLANE_C)
3585                         return 4*1024;
3586                 else
3587                         return 8*1024;
3588         }
3589 }
3590
3591 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3592 {
3593         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3594         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3595         u32 dspcntr = 0;
3596
3597         if (crtc_state->gamma_enable)
3598                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3599
3600         if (crtc_state->csc_enable)
3601                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3602
3603         if (INTEL_GEN(dev_priv) < 5)
3604                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3605
3606         return dspcntr;
3607 }
3608
3609 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3610                           const struct intel_plane_state *plane_state)
3611 {
3612         struct drm_i915_private *dev_priv =
3613                 to_i915(plane_state->base.plane->dev);
3614         const struct drm_framebuffer *fb = plane_state->base.fb;
3615         unsigned int rotation = plane_state->base.rotation;
3616         u32 dspcntr;
3617
3618         dspcntr = DISPLAY_PLANE_ENABLE;
3619
3620         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3621             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
3622                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3623
3624         switch (fb->format->format) {
3625         case DRM_FORMAT_C8:
3626                 dspcntr |= DISPPLANE_8BPP;
3627                 break;
3628         case DRM_FORMAT_XRGB1555:
3629                 dspcntr |= DISPPLANE_BGRX555;
3630                 break;
3631         case DRM_FORMAT_RGB565:
3632                 dspcntr |= DISPPLANE_BGRX565;
3633                 break;
3634         case DRM_FORMAT_XRGB8888:
3635                 dspcntr |= DISPPLANE_BGRX888;
3636                 break;
3637         case DRM_FORMAT_XBGR8888:
3638                 dspcntr |= DISPPLANE_RGBX888;
3639                 break;
3640         case DRM_FORMAT_XRGB2101010:
3641                 dspcntr |= DISPPLANE_BGRX101010;
3642                 break;
3643         case DRM_FORMAT_XBGR2101010:
3644                 dspcntr |= DISPPLANE_RGBX101010;
3645                 break;
3646         default:
3647                 MISSING_CASE(fb->format->format);
3648                 return 0;
3649         }
3650
3651         if (INTEL_GEN(dev_priv) >= 4 &&
3652             fb->modifier == I915_FORMAT_MOD_X_TILED)
3653                 dspcntr |= DISPPLANE_TILED;
3654
3655         if (rotation & DRM_MODE_ROTATE_180)
3656                 dspcntr |= DISPPLANE_ROTATE_180;
3657
3658         if (rotation & DRM_MODE_REFLECT_X)
3659                 dspcntr |= DISPPLANE_MIRROR;
3660
3661         return dspcntr;
3662 }
3663
3664 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3665 {
3666         struct drm_i915_private *dev_priv =
3667                 to_i915(plane_state->base.plane->dev);
3668         int src_x, src_y;
3669         u32 offset;
3670         int ret;
3671
3672         ret = intel_plane_compute_gtt(plane_state);
3673         if (ret)
3674                 return ret;
3675
3676         if (!plane_state->base.visible)
3677                 return 0;
3678
3679         src_x = plane_state->base.src.x1 >> 16;
3680         src_y = plane_state->base.src.y1 >> 16;
3681
3682         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3683
3684         if (INTEL_GEN(dev_priv) >= 4)
3685                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3686                                                             plane_state, 0);
3687         else
3688                 offset = 0;
3689
3690         /*
3691          * Put the final coordinates back so that the src
3692          * coordinate checks will see the right values.
3693          */
3694         drm_rect_translate(&plane_state->base.src,
3695                            (src_x << 16) - plane_state->base.src.x1,
3696                            (src_y << 16) - plane_state->base.src.y1);
3697
3698         /* HSW/BDW do this automagically in hardware */
3699         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3700                 unsigned int rotation = plane_state->base.rotation;
3701                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3702                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3703
3704                 if (rotation & DRM_MODE_ROTATE_180) {
3705                         src_x += src_w - 1;
3706                         src_y += src_h - 1;
3707                 } else if (rotation & DRM_MODE_REFLECT_X) {
3708                         src_x += src_w - 1;
3709                 }
3710         }
3711
3712         plane_state->color_plane[0].offset = offset;
3713         plane_state->color_plane[0].x = src_x;
3714         plane_state->color_plane[0].y = src_y;
3715
3716         return 0;
3717 }
3718
3719 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
3720 {
3721         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3722         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3723
3724         if (IS_CHERRYVIEW(dev_priv))
3725                 return i9xx_plane == PLANE_B;
3726         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
3727                 return false;
3728         else if (IS_GEN(dev_priv, 4))
3729                 return i9xx_plane == PLANE_C;
3730         else
3731                 return i9xx_plane == PLANE_B ||
3732                         i9xx_plane == PLANE_C;
3733 }
3734
3735 static int
3736 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3737                  struct intel_plane_state *plane_state)
3738 {
3739         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3740         int ret;
3741
3742         ret = chv_plane_check_rotation(plane_state);
3743         if (ret)
3744                 return ret;
3745
3746         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3747                                                   &crtc_state->base,
3748                                                   DRM_PLANE_HELPER_NO_SCALING,
3749                                                   DRM_PLANE_HELPER_NO_SCALING,
3750                                                   i9xx_plane_has_windowing(plane),
3751                                                   true);
3752         if (ret)
3753                 return ret;
3754
3755         ret = i9xx_check_plane_surface(plane_state);
3756         if (ret)
3757                 return ret;
3758
3759         if (!plane_state->base.visible)
3760                 return 0;
3761
3762         ret = intel_plane_check_src_coordinates(plane_state);
3763         if (ret)
3764                 return ret;
3765
3766         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3767
3768         return 0;
3769 }
3770
3771 static void i9xx_update_plane(struct intel_plane *plane,
3772                               const struct intel_crtc_state *crtc_state,
3773                               const struct intel_plane_state *plane_state)
3774 {
3775         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3776         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3777         u32 linear_offset;
3778         int x = plane_state->color_plane[0].x;
3779         int y = plane_state->color_plane[0].y;
3780         int crtc_x = plane_state->base.dst.x1;
3781         int crtc_y = plane_state->base.dst.y1;
3782         int crtc_w = drm_rect_width(&plane_state->base.dst);
3783         int crtc_h = drm_rect_height(&plane_state->base.dst);
3784         unsigned long irqflags;
3785         u32 dspaddr_offset;
3786         u32 dspcntr;
3787
3788         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
3789
3790         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3791
3792         if (INTEL_GEN(dev_priv) >= 4)
3793                 dspaddr_offset = plane_state->color_plane[0].offset;
3794         else
3795                 dspaddr_offset = linear_offset;
3796
3797         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3798
3799         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3800
3801         if (INTEL_GEN(dev_priv) < 4) {
3802                 /*
3803                  * PLANE_A doesn't actually have a full window
3804                  * generator but let's assume we still need to
3805                  * program whatever is there.
3806                  */
3807                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3808                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3809                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3810         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3811                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3812                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3813                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3814                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3815         }
3816
3817         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3818                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3819         } else if (INTEL_GEN(dev_priv) >= 4) {
3820                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3821                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3822         }
3823
3824         /*
3825          * The control register self-arms if the plane was previously
3826          * disabled. Try to make the plane enable atomic by writing
3827          * the control register just before the surface register.
3828          */
3829         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3830         if (INTEL_GEN(dev_priv) >= 4)
3831                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3832                               intel_plane_ggtt_offset(plane_state) +
3833                               dspaddr_offset);
3834         else
3835                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3836                               intel_plane_ggtt_offset(plane_state) +
3837                               dspaddr_offset);
3838
3839         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3840 }
3841
3842 static void i9xx_disable_plane(struct intel_plane *plane,
3843                                const struct intel_crtc_state *crtc_state)
3844 {
3845         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3846         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3847         unsigned long irqflags;
3848         u32 dspcntr;
3849
3850         /*
3851          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
3852          * enable on ilk+ affect the pipe bottom color as
3853          * well, so we must configure them even if the plane
3854          * is disabled.
3855          *
3856          * On pre-g4x there is no way to gamma correct the
3857          * pipe bottom color but we'll keep on doing this
3858          * anyway so that the crtc state readout works correctly.
3859          */
3860         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
3861
3862         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3863
3864         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3865         if (INTEL_GEN(dev_priv) >= 4)
3866                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3867         else
3868                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3869
3870         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3871 }
3872
3873 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3874                                     enum pipe *pipe)
3875 {
3876         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3877         enum intel_display_power_domain power_domain;
3878         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3879         intel_wakeref_t wakeref;
3880         bool ret;
3881         u32 val;
3882
3883         /*
3884          * Not 100% correct for planes that can move between pipes,
3885          * but that's only the case for gen2-4 which don't have any
3886          * display power wells.
3887          */
3888         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3889         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3890         if (!wakeref)
3891                 return false;
3892
3893         val = I915_READ(DSPCNTR(i9xx_plane));
3894
3895         ret = val & DISPLAY_PLANE_ENABLE;
3896
3897         if (INTEL_GEN(dev_priv) >= 5)
3898                 *pipe = plane->pipe;
3899         else
3900                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3901                         DISPPLANE_SEL_PIPE_SHIFT;
3902
3903         intel_display_power_put(dev_priv, power_domain, wakeref);
3904
3905         return ret;
3906 }
3907
3908 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3909 {
3910         struct drm_device *dev = intel_crtc->base.dev;
3911         struct drm_i915_private *dev_priv = to_i915(dev);
3912
3913         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3914         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3915         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3916 }
3917
3918 /*
3919  * This function detaches (aka. unbinds) unused scalers in hardware
3920  */
3921 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3922 {
3923         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3924         const struct intel_crtc_scaler_state *scaler_state =
3925                 &crtc_state->scaler_state;
3926         int i;
3927
3928         /* loop through and disable scalers that aren't in use */
3929         for (i = 0; i < intel_crtc->num_scalers; i++) {
3930                 if (!scaler_state->scalers[i].in_use)
3931                         skl_detach_scaler(intel_crtc, i);
3932         }
3933 }
3934
3935 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3936                                           int color_plane, unsigned int rotation)
3937 {
3938         /*
3939          * The stride is either expressed as a multiple of 64 bytes chunks for
3940          * linear buffers or in number of tiles for tiled buffers.
3941          */
3942         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3943                 return 64;
3944         else if (drm_rotation_90_or_270(rotation))
3945                 return intel_tile_height(fb, color_plane);
3946         else
3947                 return intel_tile_width_bytes(fb, color_plane);
3948 }
3949
3950 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3951                      int color_plane)
3952 {
3953         const struct drm_framebuffer *fb = plane_state->base.fb;
3954         unsigned int rotation = plane_state->base.rotation;
3955         u32 stride = plane_state->color_plane[color_plane].stride;
3956
3957         if (color_plane >= fb->format->num_planes)
3958                 return 0;
3959
3960         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3961 }
3962
3963 static u32 skl_plane_ctl_format(u32 pixel_format)
3964 {
3965         switch (pixel_format) {
3966         case DRM_FORMAT_C8:
3967                 return PLANE_CTL_FORMAT_INDEXED;
3968         case DRM_FORMAT_RGB565:
3969                 return PLANE_CTL_FORMAT_RGB_565;
3970         case DRM_FORMAT_XBGR8888:
3971         case DRM_FORMAT_ABGR8888:
3972                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3973         case DRM_FORMAT_XRGB8888:
3974         case DRM_FORMAT_ARGB8888:
3975                 return PLANE_CTL_FORMAT_XRGB_8888;
3976         case DRM_FORMAT_XBGR2101010:
3977                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
3978         case DRM_FORMAT_XRGB2101010:
3979                 return PLANE_CTL_FORMAT_XRGB_2101010;
3980         case DRM_FORMAT_XBGR16161616F:
3981         case DRM_FORMAT_ABGR16161616F:
3982                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
3983         case DRM_FORMAT_XRGB16161616F:
3984         case DRM_FORMAT_ARGB16161616F:
3985                 return PLANE_CTL_FORMAT_XRGB_16161616F;
3986         case DRM_FORMAT_YUYV:
3987                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3988         case DRM_FORMAT_YVYU:
3989                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3990         case DRM_FORMAT_UYVY:
3991                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3992         case DRM_FORMAT_VYUY:
3993                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3994         case DRM_FORMAT_NV12:
3995                 return PLANE_CTL_FORMAT_NV12;
3996         case DRM_FORMAT_P010:
3997                 return PLANE_CTL_FORMAT_P010;
3998         case DRM_FORMAT_P012:
3999                 return PLANE_CTL_FORMAT_P012;
4000         case DRM_FORMAT_P016:
4001                 return PLANE_CTL_FORMAT_P016;
4002         case DRM_FORMAT_Y210:
4003                 return PLANE_CTL_FORMAT_Y210;
4004         case DRM_FORMAT_Y212:
4005                 return PLANE_CTL_FORMAT_Y212;
4006         case DRM_FORMAT_Y216:
4007                 return PLANE_CTL_FORMAT_Y216;
4008         case DRM_FORMAT_XVYU2101010:
4009                 return PLANE_CTL_FORMAT_Y410;
4010         case DRM_FORMAT_XVYU12_16161616:
4011                 return PLANE_CTL_FORMAT_Y412;
4012         case DRM_FORMAT_XVYU16161616:
4013                 return PLANE_CTL_FORMAT_Y416;
4014         default:
4015                 MISSING_CASE(pixel_format);
4016         }
4017
4018         return 0;
4019 }
4020
4021 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4022 {
4023         if (!plane_state->base.fb->format->has_alpha)
4024                 return PLANE_CTL_ALPHA_DISABLE;
4025
4026         switch (plane_state->base.pixel_blend_mode) {
4027         case DRM_MODE_BLEND_PIXEL_NONE:
4028                 return PLANE_CTL_ALPHA_DISABLE;
4029         case DRM_MODE_BLEND_PREMULTI:
4030                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4031         case DRM_MODE_BLEND_COVERAGE:
4032                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4033         default:
4034                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4035                 return PLANE_CTL_ALPHA_DISABLE;
4036         }
4037 }
4038
4039 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4040 {
4041         if (!plane_state->base.fb->format->has_alpha)
4042                 return PLANE_COLOR_ALPHA_DISABLE;
4043
4044         switch (plane_state->base.pixel_blend_mode) {
4045         case DRM_MODE_BLEND_PIXEL_NONE:
4046                 return PLANE_COLOR_ALPHA_DISABLE;
4047         case DRM_MODE_BLEND_PREMULTI:
4048                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4049         case DRM_MODE_BLEND_COVERAGE:
4050                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4051         default:
4052                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4053                 return PLANE_COLOR_ALPHA_DISABLE;
4054         }
4055 }
4056
4057 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4058 {
4059         switch (fb_modifier) {
4060         case DRM_FORMAT_MOD_LINEAR:
4061                 break;
4062         case I915_FORMAT_MOD_X_TILED:
4063                 return PLANE_CTL_TILED_X;
4064         case I915_FORMAT_MOD_Y_TILED:
4065                 return PLANE_CTL_TILED_Y;
4066         case I915_FORMAT_MOD_Y_TILED_CCS:
4067                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4068         case I915_FORMAT_MOD_Yf_TILED:
4069                 return PLANE_CTL_TILED_YF;
4070         case I915_FORMAT_MOD_Yf_TILED_CCS:
4071                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4072         default:
4073                 MISSING_CASE(fb_modifier);
4074         }
4075
4076         return 0;
4077 }
4078
4079 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4080 {
4081         switch (rotate) {
4082         case DRM_MODE_ROTATE_0:
4083                 break;
4084         /*
4085          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4086          * while i915 HW rotation is clockwise, thats why this swapping.
4087          */
4088         case DRM_MODE_ROTATE_90:
4089                 return PLANE_CTL_ROTATE_270;
4090         case DRM_MODE_ROTATE_180:
4091                 return PLANE_CTL_ROTATE_180;
4092         case DRM_MODE_ROTATE_270:
4093                 return PLANE_CTL_ROTATE_90;
4094         default:
4095                 MISSING_CASE(rotate);
4096         }
4097
4098         return 0;
4099 }
4100
4101 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4102 {
4103         switch (reflect) {
4104         case 0:
4105                 break;
4106         case DRM_MODE_REFLECT_X:
4107                 return PLANE_CTL_FLIP_HORIZONTAL;
4108         case DRM_MODE_REFLECT_Y:
4109         default:
4110                 MISSING_CASE(reflect);
4111         }
4112
4113         return 0;
4114 }
4115
4116 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4117 {
4118         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4119         u32 plane_ctl = 0;
4120
4121         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4122                 return plane_ctl;
4123
4124         if (crtc_state->gamma_enable)
4125                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4126
4127         if (crtc_state->csc_enable)
4128                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4129
4130         return plane_ctl;
4131 }
4132
4133 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4134                   const struct intel_plane_state *plane_state)
4135 {
4136         struct drm_i915_private *dev_priv =
4137                 to_i915(plane_state->base.plane->dev);
4138         const struct drm_framebuffer *fb = plane_state->base.fb;
4139         unsigned int rotation = plane_state->base.rotation;
4140         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4141         u32 plane_ctl;
4142
4143         plane_ctl = PLANE_CTL_ENABLE;
4144
4145         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4146                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4147                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4148
4149                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4150                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4151
4152                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4153                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4154         }
4155
4156         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4157         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4158         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4159
4160         if (INTEL_GEN(dev_priv) >= 10)
4161                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4162                                                 DRM_MODE_REFLECT_MASK);
4163
4164         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4165                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4166         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4167                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4168
4169         return plane_ctl;
4170 }
4171
4172 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4173 {
4174         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4175         u32 plane_color_ctl = 0;
4176
4177         if (INTEL_GEN(dev_priv) >= 11)
4178                 return plane_color_ctl;
4179
4180         if (crtc_state->gamma_enable)
4181                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4182
4183         if (crtc_state->csc_enable)
4184                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4185
4186         return plane_color_ctl;
4187 }
4188
4189 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4190                         const struct intel_plane_state *plane_state)
4191 {
4192         struct drm_i915_private *dev_priv =
4193                 to_i915(plane_state->base.plane->dev);
4194         const struct drm_framebuffer *fb = plane_state->base.fb;
4195         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4196         u32 plane_color_ctl = 0;
4197
4198         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4199         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4200
4201         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4202                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4203                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4204                 else
4205                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4206
4207                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4208                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4209         } else if (fb->format->is_yuv) {
4210                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4211         }
4212
4213         return plane_color_ctl;
4214 }
4215
4216 static int
4217 __intel_display_resume(struct drm_device *dev,
4218                        struct drm_atomic_state *state,
4219                        struct drm_modeset_acquire_ctx *ctx)
4220 {
4221         struct drm_crtc_state *crtc_state;
4222         struct drm_crtc *crtc;
4223         int i, ret;
4224
4225         intel_modeset_setup_hw_state(dev, ctx);
4226         i915_redisable_vga(to_i915(dev));
4227
4228         if (!state)
4229                 return 0;
4230
4231         /*
4232          * We've duplicated the state, pointers to the old state are invalid.
4233          *
4234          * Don't attempt to use the old state until we commit the duplicated state.
4235          */
4236         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4237                 /*
4238                  * Force recalculation even if we restore
4239                  * current state. With fast modeset this may not result
4240                  * in a modeset when the state is compatible.
4241                  */
4242                 crtc_state->mode_changed = true;
4243         }
4244
4245         /* ignore any reset values/BIOS leftovers in the WM registers */
4246         if (!HAS_GMCH(to_i915(dev)))
4247                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4248
4249         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4250
4251         WARN_ON(ret == -EDEADLK);
4252         return ret;
4253 }
4254
4255 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4256 {
4257         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4258                 intel_has_gpu_reset(dev_priv));
4259 }
4260
4261 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4262 {
4263         struct drm_device *dev = &dev_priv->drm;
4264         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4265         struct drm_atomic_state *state;
4266         int ret;
4267
4268         /* reset doesn't touch the display */
4269         if (!i915_modparams.force_reset_modeset_test &&
4270             !gpu_reset_clobbers_display(dev_priv))
4271                 return;
4272
4273         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4274         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
4275         wake_up_all(&dev_priv->gpu_error.wait_queue);
4276
4277         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4278                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4279                 i915_gem_set_wedged(dev_priv);
4280         }
4281
4282         /*
4283          * Need mode_config.mutex so that we don't
4284          * trample ongoing ->detect() and whatnot.
4285          */
4286         mutex_lock(&dev->mode_config.mutex);
4287         drm_modeset_acquire_init(ctx, 0);
4288         while (1) {
4289                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4290                 if (ret != -EDEADLK)
4291                         break;
4292
4293                 drm_modeset_backoff(ctx);
4294         }
4295         /*
4296          * Disabling the crtcs gracefully seems nicer. Also the
4297          * g33 docs say we should at least disable all the planes.
4298          */
4299         state = drm_atomic_helper_duplicate_state(dev, ctx);
4300         if (IS_ERR(state)) {
4301                 ret = PTR_ERR(state);
4302                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4303                 return;
4304         }
4305
4306         ret = drm_atomic_helper_disable_all(dev, ctx);
4307         if (ret) {
4308                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4309                 drm_atomic_state_put(state);
4310                 return;
4311         }
4312
4313         dev_priv->modeset_restore_state = state;
4314         state->acquire_ctx = ctx;
4315 }
4316
4317 void intel_finish_reset(struct drm_i915_private *dev_priv)
4318 {
4319         struct drm_device *dev = &dev_priv->drm;
4320         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4321         struct drm_atomic_state *state;
4322         int ret;
4323
4324         /* reset doesn't touch the display */
4325         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
4326                 return;
4327
4328         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4329         if (!state)
4330                 goto unlock;
4331
4332         /* reset doesn't touch the display */
4333         if (!gpu_reset_clobbers_display(dev_priv)) {
4334                 /* for testing only restore the display */
4335                 ret = __intel_display_resume(dev, state, ctx);
4336                 if (ret)
4337                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4338         } else {
4339                 /*
4340                  * The display has been reset as well,
4341                  * so need a full re-initialization.
4342                  */
4343                 intel_pps_unlock_regs_wa(dev_priv);
4344                 intel_modeset_init_hw(dev);
4345                 intel_init_clock_gating(dev_priv);
4346
4347                 spin_lock_irq(&dev_priv->irq_lock);
4348                 if (dev_priv->display.hpd_irq_setup)
4349                         dev_priv->display.hpd_irq_setup(dev_priv);
4350                 spin_unlock_irq(&dev_priv->irq_lock);
4351
4352                 ret = __intel_display_resume(dev, state, ctx);
4353                 if (ret)
4354                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4355
4356                 intel_hpd_init(dev_priv);
4357         }
4358
4359         drm_atomic_state_put(state);
4360 unlock:
4361         drm_modeset_drop_locks(ctx);
4362         drm_modeset_acquire_fini(ctx);
4363         mutex_unlock(&dev->mode_config.mutex);
4364
4365         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
4366 }
4367
4368 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4369 {
4370         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4371         enum pipe pipe = crtc->pipe;
4372         u32 tmp;
4373
4374         tmp = I915_READ(PIPE_CHICKEN(pipe));
4375
4376         /*
4377          * Display WA #1153: icl
4378          * enable hardware to bypass the alpha math
4379          * and rounding for per-pixel values 00 and 0xff
4380          */
4381         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4382         /*
4383          * Display WA # 1605353570: icl
4384          * Set the pixel rounding bit to 1 for allowing
4385          * passthrough of Frame buffer pixels unmodified
4386          * across pipe
4387          */
4388         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4389         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4390 }
4391
4392 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
4393                                      const struct intel_crtc_state *new_crtc_state)
4394 {
4395         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
4396         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4397
4398         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
4399         crtc->base.mode = new_crtc_state->base.mode;
4400
4401         /*
4402          * Update pipe size and adjust fitter if needed: the reason for this is
4403          * that in compute_mode_changes we check the native mode (not the pfit
4404          * mode) to see if we can flip rather than do a full mode set. In the
4405          * fastboot case, we'll flip, but if we don't update the pipesrc and
4406          * pfit state, we'll end up with a big fb scanned out into the wrong
4407          * sized surface.
4408          */
4409
4410         I915_WRITE(PIPESRC(crtc->pipe),
4411                    ((new_crtc_state->pipe_src_w - 1) << 16) |
4412                    (new_crtc_state->pipe_src_h - 1));
4413
4414         /* on skylake this is done by detaching scalers */
4415         if (INTEL_GEN(dev_priv) >= 9) {
4416                 skl_detach_scalers(new_crtc_state);
4417
4418                 if (new_crtc_state->pch_pfit.enabled)
4419                         skylake_pfit_enable(new_crtc_state);
4420         } else if (HAS_PCH_SPLIT(dev_priv)) {
4421                 if (new_crtc_state->pch_pfit.enabled)
4422                         ironlake_pfit_enable(new_crtc_state);
4423                 else if (old_crtc_state->pch_pfit.enabled)
4424                         ironlake_pfit_disable(old_crtc_state);
4425         }
4426
4427         if (INTEL_GEN(dev_priv) >= 11)
4428                 icl_set_pipe_chicken(crtc);
4429 }
4430
4431 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4432 {
4433         struct drm_device *dev = crtc->base.dev;
4434         struct drm_i915_private *dev_priv = to_i915(dev);
4435         int pipe = crtc->pipe;
4436         i915_reg_t reg;
4437         u32 temp;
4438
4439         /* enable normal train */
4440         reg = FDI_TX_CTL(pipe);
4441         temp = I915_READ(reg);
4442         if (IS_IVYBRIDGE(dev_priv)) {
4443                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4444                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4445         } else {
4446                 temp &= ~FDI_LINK_TRAIN_NONE;
4447                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4448         }
4449         I915_WRITE(reg, temp);
4450
4451         reg = FDI_RX_CTL(pipe);
4452         temp = I915_READ(reg);
4453         if (HAS_PCH_CPT(dev_priv)) {
4454                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4455                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4456         } else {
4457                 temp &= ~FDI_LINK_TRAIN_NONE;
4458                 temp |= FDI_LINK_TRAIN_NONE;
4459         }
4460         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4461
4462         /* wait one idle pattern time */
4463         POSTING_READ(reg);
4464         udelay(1000);
4465
4466         /* IVB wants error correction enabled */
4467         if (IS_IVYBRIDGE(dev_priv))
4468                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4469                            FDI_FE_ERRC_ENABLE);
4470 }
4471
4472 /* The FDI link training functions for ILK/Ibexpeak. */
4473 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4474                                     const struct intel_crtc_state *crtc_state)
4475 {
4476         struct drm_device *dev = crtc->base.dev;
4477         struct drm_i915_private *dev_priv = to_i915(dev);
4478         int pipe = crtc->pipe;
4479         i915_reg_t reg;
4480         u32 temp, tries;
4481
4482         /* FDI needs bits from pipe first */
4483         assert_pipe_enabled(dev_priv, pipe);
4484
4485         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4486            for train result */
4487         reg = FDI_RX_IMR(pipe);
4488         temp = I915_READ(reg);
4489         temp &= ~FDI_RX_SYMBOL_LOCK;
4490         temp &= ~FDI_RX_BIT_LOCK;
4491         I915_WRITE(reg, temp);
4492         I915_READ(reg);
4493         udelay(150);
4494
4495         /* enable CPU FDI TX and PCH FDI RX */
4496         reg = FDI_TX_CTL(pipe);
4497         temp = I915_READ(reg);
4498         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4499         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4500         temp &= ~FDI_LINK_TRAIN_NONE;
4501         temp |= FDI_LINK_TRAIN_PATTERN_1;
4502         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4503
4504         reg = FDI_RX_CTL(pipe);
4505         temp = I915_READ(reg);
4506         temp &= ~FDI_LINK_TRAIN_NONE;
4507         temp |= FDI_LINK_TRAIN_PATTERN_1;
4508         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4509
4510         POSTING_READ(reg);
4511         udelay(150);
4512
4513         /* Ironlake workaround, enable clock pointer after FDI enable*/
4514         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4515         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4516                    FDI_RX_PHASE_SYNC_POINTER_EN);
4517
4518         reg = FDI_RX_IIR(pipe);
4519         for (tries = 0; tries < 5; tries++) {
4520                 temp = I915_READ(reg);
4521                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4522
4523                 if ((temp & FDI_RX_BIT_LOCK)) {
4524                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4525                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4526                         break;
4527                 }
4528         }
4529         if (tries == 5)
4530                 DRM_ERROR("FDI train 1 fail!\n");
4531
4532         /* Train 2 */
4533         reg = FDI_TX_CTL(pipe);
4534         temp = I915_READ(reg);
4535         temp &= ~FDI_LINK_TRAIN_NONE;
4536         temp |= FDI_LINK_TRAIN_PATTERN_2;
4537         I915_WRITE(reg, temp);
4538
4539         reg = FDI_RX_CTL(pipe);
4540         temp = I915_READ(reg);
4541         temp &= ~FDI_LINK_TRAIN_NONE;
4542         temp |= FDI_LINK_TRAIN_PATTERN_2;
4543         I915_WRITE(reg, temp);
4544
4545         POSTING_READ(reg);
4546         udelay(150);
4547
4548         reg = FDI_RX_IIR(pipe);
4549         for (tries = 0; tries < 5; tries++) {
4550                 temp = I915_READ(reg);
4551                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4552
4553                 if (temp & FDI_RX_SYMBOL_LOCK) {
4554                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4555                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4556                         break;
4557                 }
4558         }
4559         if (tries == 5)
4560                 DRM_ERROR("FDI train 2 fail!\n");
4561
4562         DRM_DEBUG_KMS("FDI train done\n");
4563
4564 }
4565
4566 static const int snb_b_fdi_train_param[] = {
4567         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4568         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4569         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4570         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4571 };
4572
4573 /* The FDI link training functions for SNB/Cougarpoint. */
4574 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4575                                 const struct intel_crtc_state *crtc_state)
4576 {
4577         struct drm_device *dev = crtc->base.dev;
4578         struct drm_i915_private *dev_priv = to_i915(dev);
4579         int pipe = crtc->pipe;
4580         i915_reg_t reg;
4581         u32 temp, i, retry;
4582
4583         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4584            for train result */
4585         reg = FDI_RX_IMR(pipe);
4586         temp = I915_READ(reg);
4587         temp &= ~FDI_RX_SYMBOL_LOCK;
4588         temp &= ~FDI_RX_BIT_LOCK;
4589         I915_WRITE(reg, temp);
4590
4591         POSTING_READ(reg);
4592         udelay(150);
4593
4594         /* enable CPU FDI TX and PCH FDI RX */
4595         reg = FDI_TX_CTL(pipe);
4596         temp = I915_READ(reg);
4597         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4598         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4599         temp &= ~FDI_LINK_TRAIN_NONE;
4600         temp |= FDI_LINK_TRAIN_PATTERN_1;
4601         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4602         /* SNB-B */
4603         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4604         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4605
4606         I915_WRITE(FDI_RX_MISC(pipe),
4607                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4608
4609         reg = FDI_RX_CTL(pipe);
4610         temp = I915_READ(reg);
4611         if (HAS_PCH_CPT(dev_priv)) {
4612                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4613                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4614         } else {
4615                 temp &= ~FDI_LINK_TRAIN_NONE;
4616                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4617         }
4618         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4619
4620         POSTING_READ(reg);
4621         udelay(150);
4622
4623         for (i = 0; i < 4; i++) {
4624                 reg = FDI_TX_CTL(pipe);
4625                 temp = I915_READ(reg);
4626                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4627                 temp |= snb_b_fdi_train_param[i];
4628                 I915_WRITE(reg, temp);
4629
4630                 POSTING_READ(reg);
4631                 udelay(500);
4632
4633                 for (retry = 0; retry < 5; retry++) {
4634                         reg = FDI_RX_IIR(pipe);
4635                         temp = I915_READ(reg);
4636                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4637                         if (temp & FDI_RX_BIT_LOCK) {
4638                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4639                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4640                                 break;
4641                         }
4642                         udelay(50);
4643                 }
4644                 if (retry < 5)
4645                         break;
4646         }
4647         if (i == 4)
4648                 DRM_ERROR("FDI train 1 fail!\n");
4649
4650         /* Train 2 */
4651         reg = FDI_TX_CTL(pipe);
4652         temp = I915_READ(reg);
4653         temp &= ~FDI_LINK_TRAIN_NONE;
4654         temp |= FDI_LINK_TRAIN_PATTERN_2;
4655         if (IS_GEN(dev_priv, 6)) {
4656                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4657                 /* SNB-B */
4658                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4659         }
4660         I915_WRITE(reg, temp);
4661
4662         reg = FDI_RX_CTL(pipe);
4663         temp = I915_READ(reg);
4664         if (HAS_PCH_CPT(dev_priv)) {
4665                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4666                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4667         } else {
4668                 temp &= ~FDI_LINK_TRAIN_NONE;
4669                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4670         }
4671         I915_WRITE(reg, temp);
4672
4673         POSTING_READ(reg);
4674         udelay(150);
4675
4676         for (i = 0; i < 4; i++) {
4677                 reg = FDI_TX_CTL(pipe);
4678                 temp = I915_READ(reg);
4679                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4680                 temp |= snb_b_fdi_train_param[i];
4681                 I915_WRITE(reg, temp);
4682
4683                 POSTING_READ(reg);
4684                 udelay(500);
4685
4686                 for (retry = 0; retry < 5; retry++) {
4687                         reg = FDI_RX_IIR(pipe);
4688                         temp = I915_READ(reg);
4689                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4690                         if (temp & FDI_RX_SYMBOL_LOCK) {
4691                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4692                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4693                                 break;
4694                         }
4695                         udelay(50);
4696                 }
4697                 if (retry < 5)
4698                         break;
4699         }
4700         if (i == 4)
4701                 DRM_ERROR("FDI train 2 fail!\n");
4702
4703         DRM_DEBUG_KMS("FDI train done.\n");
4704 }
4705
4706 /* Manual link training for Ivy Bridge A0 parts */
4707 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4708                                       const struct intel_crtc_state *crtc_state)
4709 {
4710         struct drm_device *dev = crtc->base.dev;
4711         struct drm_i915_private *dev_priv = to_i915(dev);
4712         int pipe = crtc->pipe;
4713         i915_reg_t reg;
4714         u32 temp, i, j;
4715
4716         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4717            for train result */
4718         reg = FDI_RX_IMR(pipe);
4719         temp = I915_READ(reg);
4720         temp &= ~FDI_RX_SYMBOL_LOCK;
4721         temp &= ~FDI_RX_BIT_LOCK;
4722         I915_WRITE(reg, temp);
4723
4724         POSTING_READ(reg);
4725         udelay(150);
4726
4727         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4728                       I915_READ(FDI_RX_IIR(pipe)));
4729
4730         /* Try each vswing and preemphasis setting twice before moving on */
4731         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4732                 /* disable first in case we need to retry */
4733                 reg = FDI_TX_CTL(pipe);
4734                 temp = I915_READ(reg);
4735                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4736                 temp &= ~FDI_TX_ENABLE;
4737                 I915_WRITE(reg, temp);
4738
4739                 reg = FDI_RX_CTL(pipe);
4740                 temp = I915_READ(reg);
4741                 temp &= ~FDI_LINK_TRAIN_AUTO;
4742                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4743                 temp &= ~FDI_RX_ENABLE;
4744                 I915_WRITE(reg, temp);
4745
4746                 /* enable CPU FDI TX and PCH FDI RX */
4747                 reg = FDI_TX_CTL(pipe);
4748                 temp = I915_READ(reg);
4749                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4750                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4751                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4752                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4753                 temp |= snb_b_fdi_train_param[j/2];
4754                 temp |= FDI_COMPOSITE_SYNC;
4755                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4756
4757                 I915_WRITE(FDI_RX_MISC(pipe),
4758                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4759
4760                 reg = FDI_RX_CTL(pipe);
4761                 temp = I915_READ(reg);
4762                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4763                 temp |= FDI_COMPOSITE_SYNC;
4764                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4765
4766                 POSTING_READ(reg);
4767                 udelay(1); /* should be 0.5us */
4768
4769                 for (i = 0; i < 4; i++) {
4770                         reg = FDI_RX_IIR(pipe);
4771                         temp = I915_READ(reg);
4772                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4773
4774                         if (temp & FDI_RX_BIT_LOCK ||
4775                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4776                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4777                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4778                                               i);
4779                                 break;
4780                         }
4781                         udelay(1); /* should be 0.5us */
4782                 }
4783                 if (i == 4) {
4784                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4785                         continue;
4786                 }
4787
4788                 /* Train 2 */
4789                 reg = FDI_TX_CTL(pipe);
4790                 temp = I915_READ(reg);
4791                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4792                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4793                 I915_WRITE(reg, temp);
4794
4795                 reg = FDI_RX_CTL(pipe);
4796                 temp = I915_READ(reg);
4797                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4798                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4799                 I915_WRITE(reg, temp);
4800
4801                 POSTING_READ(reg);
4802                 udelay(2); /* should be 1.5us */
4803
4804                 for (i = 0; i < 4; i++) {
4805                         reg = FDI_RX_IIR(pipe);
4806                         temp = I915_READ(reg);
4807                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4808
4809                         if (temp & FDI_RX_SYMBOL_LOCK ||
4810                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4811                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4812                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4813                                               i);
4814                                 goto train_done;
4815                         }
4816                         udelay(2); /* should be 1.5us */
4817                 }
4818                 if (i == 4)
4819                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4820         }
4821
4822 train_done:
4823         DRM_DEBUG_KMS("FDI train done.\n");
4824 }
4825
4826 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4827 {
4828         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4829         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4830         int pipe = intel_crtc->pipe;
4831         i915_reg_t reg;
4832         u32 temp;
4833
4834         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4835         reg = FDI_RX_CTL(pipe);
4836         temp = I915_READ(reg);
4837         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4838         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4839         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4840         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4841
4842         POSTING_READ(reg);
4843         udelay(200);
4844
4845         /* Switch from Rawclk to PCDclk */
4846         temp = I915_READ(reg);
4847         I915_WRITE(reg, temp | FDI_PCDCLK);
4848
4849         POSTING_READ(reg);
4850         udelay(200);
4851
4852         /* Enable CPU FDI TX PLL, always on for Ironlake */
4853         reg = FDI_TX_CTL(pipe);
4854         temp = I915_READ(reg);
4855         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4856                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4857
4858                 POSTING_READ(reg);
4859                 udelay(100);
4860         }
4861 }
4862
4863 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4864 {
4865         struct drm_device *dev = intel_crtc->base.dev;
4866         struct drm_i915_private *dev_priv = to_i915(dev);
4867         int pipe = intel_crtc->pipe;
4868         i915_reg_t reg;
4869         u32 temp;
4870
4871         /* Switch from PCDclk to Rawclk */
4872         reg = FDI_RX_CTL(pipe);
4873         temp = I915_READ(reg);
4874         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4875
4876         /* Disable CPU FDI TX PLL */
4877         reg = FDI_TX_CTL(pipe);
4878         temp = I915_READ(reg);
4879         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4880
4881         POSTING_READ(reg);
4882         udelay(100);
4883
4884         reg = FDI_RX_CTL(pipe);
4885         temp = I915_READ(reg);
4886         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4887
4888         /* Wait for the clocks to turn off. */
4889         POSTING_READ(reg);
4890         udelay(100);
4891 }
4892
4893 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4894 {
4895         struct drm_device *dev = crtc->dev;
4896         struct drm_i915_private *dev_priv = to_i915(dev);
4897         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4898         int pipe = intel_crtc->pipe;
4899         i915_reg_t reg;
4900         u32 temp;
4901
4902         /* disable CPU FDI tx and PCH FDI rx */
4903         reg = FDI_TX_CTL(pipe);
4904         temp = I915_READ(reg);
4905         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4906         POSTING_READ(reg);
4907
4908         reg = FDI_RX_CTL(pipe);
4909         temp = I915_READ(reg);
4910         temp &= ~(0x7 << 16);
4911         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4912         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4913
4914         POSTING_READ(reg);
4915         udelay(100);
4916
4917         /* Ironlake workaround, disable clock pointer after downing FDI */
4918         if (HAS_PCH_IBX(dev_priv))
4919                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4920
4921         /* still set train pattern 1 */
4922         reg = FDI_TX_CTL(pipe);
4923         temp = I915_READ(reg);
4924         temp &= ~FDI_LINK_TRAIN_NONE;
4925         temp |= FDI_LINK_TRAIN_PATTERN_1;
4926         I915_WRITE(reg, temp);
4927
4928         reg = FDI_RX_CTL(pipe);
4929         temp = I915_READ(reg);
4930         if (HAS_PCH_CPT(dev_priv)) {
4931                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4932                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4933         } else {
4934                 temp &= ~FDI_LINK_TRAIN_NONE;
4935                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4936         }
4937         /* BPC in FDI rx is consistent with that in PIPECONF */
4938         temp &= ~(0x07 << 16);
4939         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4940         I915_WRITE(reg, temp);
4941
4942         POSTING_READ(reg);
4943         udelay(100);
4944 }
4945
4946 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4947 {
4948         struct drm_crtc *crtc;
4949         bool cleanup_done;
4950
4951         drm_for_each_crtc(crtc, &dev_priv->drm) {
4952                 struct drm_crtc_commit *commit;
4953                 spin_lock(&crtc->commit_lock);
4954                 commit = list_first_entry_or_null(&crtc->commit_list,
4955                                                   struct drm_crtc_commit, commit_entry);
4956                 cleanup_done = commit ?
4957                         try_wait_for_completion(&commit->cleanup_done) : true;
4958                 spin_unlock(&crtc->commit_lock);
4959
4960                 if (cleanup_done)
4961                         continue;
4962
4963                 drm_crtc_wait_one_vblank(crtc);
4964
4965                 return true;
4966         }
4967
4968         return false;
4969 }
4970
4971 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4972 {
4973         u32 temp;
4974
4975         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4976
4977         mutex_lock(&dev_priv->sb_lock);
4978
4979         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4980         temp |= SBI_SSCCTL_DISABLE;
4981         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4982
4983         mutex_unlock(&dev_priv->sb_lock);
4984 }
4985
4986 /* Program iCLKIP clock to the desired frequency */
4987 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4988 {
4989         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4990         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4991         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4992         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4993         u32 temp;
4994
4995         lpt_disable_iclkip(dev_priv);
4996
4997         /* The iCLK virtual clock root frequency is in MHz,
4998          * but the adjusted_mode->crtc_clock in in KHz. To get the
4999          * divisors, it is necessary to divide one by another, so we
5000          * convert the virtual clock precision to KHz here for higher
5001          * precision.
5002          */
5003         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5004                 u32 iclk_virtual_root_freq = 172800 * 1000;
5005                 u32 iclk_pi_range = 64;
5006                 u32 desired_divisor;
5007
5008                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5009                                                     clock << auxdiv);
5010                 divsel = (desired_divisor / iclk_pi_range) - 2;
5011                 phaseinc = desired_divisor % iclk_pi_range;
5012
5013                 /*
5014                  * Near 20MHz is a corner case which is
5015                  * out of range for the 7-bit divisor
5016                  */
5017                 if (divsel <= 0x7f)
5018                         break;
5019         }
5020
5021         /* This should not happen with any sane values */
5022         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5023                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5024         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5025                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5026
5027         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5028                         clock,
5029                         auxdiv,
5030                         divsel,
5031                         phasedir,
5032                         phaseinc);
5033
5034         mutex_lock(&dev_priv->sb_lock);
5035
5036         /* Program SSCDIVINTPHASE6 */
5037         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5038         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5039         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5040         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5041         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5042         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5043         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5044         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5045
5046         /* Program SSCAUXDIV */
5047         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5048         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5049         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5050         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5051
5052         /* Enable modulator and associated divider */
5053         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5054         temp &= ~SBI_SSCCTL_DISABLE;
5055         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5056
5057         mutex_unlock(&dev_priv->sb_lock);
5058
5059         /* Wait for initialization time */
5060         udelay(24);
5061
5062         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5063 }
5064
5065 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5066 {
5067         u32 divsel, phaseinc, auxdiv;
5068         u32 iclk_virtual_root_freq = 172800 * 1000;
5069         u32 iclk_pi_range = 64;
5070         u32 desired_divisor;
5071         u32 temp;
5072
5073         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5074                 return 0;
5075
5076         mutex_lock(&dev_priv->sb_lock);
5077
5078         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5079         if (temp & SBI_SSCCTL_DISABLE) {
5080                 mutex_unlock(&dev_priv->sb_lock);
5081                 return 0;
5082         }
5083
5084         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5085         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5086                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5087         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5088                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5089
5090         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5091         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5092                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5093
5094         mutex_unlock(&dev_priv->sb_lock);
5095
5096         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5097
5098         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5099                                  desired_divisor << auxdiv);
5100 }
5101
5102 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5103                                                 enum pipe pch_transcoder)
5104 {
5105         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5106         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5107         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5108
5109         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5110                    I915_READ(HTOTAL(cpu_transcoder)));
5111         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5112                    I915_READ(HBLANK(cpu_transcoder)));
5113         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5114                    I915_READ(HSYNC(cpu_transcoder)));
5115
5116         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5117                    I915_READ(VTOTAL(cpu_transcoder)));
5118         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5119                    I915_READ(VBLANK(cpu_transcoder)));
5120         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5121                    I915_READ(VSYNC(cpu_transcoder)));
5122         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5123                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5124 }
5125
5126 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5127 {
5128         u32 temp;
5129
5130         temp = I915_READ(SOUTH_CHICKEN1);
5131         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5132                 return;
5133
5134         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5135         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5136
5137         temp &= ~FDI_BC_BIFURCATION_SELECT;
5138         if (enable)
5139                 temp |= FDI_BC_BIFURCATION_SELECT;
5140
5141         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5142         I915_WRITE(SOUTH_CHICKEN1, temp);
5143         POSTING_READ(SOUTH_CHICKEN1);
5144 }
5145
5146 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5147 {
5148         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5149         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5150
5151         switch (crtc->pipe) {
5152         case PIPE_A:
5153                 break;
5154         case PIPE_B:
5155                 if (crtc_state->fdi_lanes > 2)
5156                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5157                 else
5158                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5159
5160                 break;
5161         case PIPE_C:
5162                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5163
5164                 break;
5165         default:
5166                 BUG();
5167         }
5168 }
5169
5170 /*
5171  * Finds the encoder associated with the given CRTC. This can only be
5172  * used when we know that the CRTC isn't feeding multiple encoders!
5173  */
5174 static struct intel_encoder *
5175 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5176                            const struct intel_crtc_state *crtc_state)
5177 {
5178         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5179         const struct drm_connector_state *connector_state;
5180         const struct drm_connector *connector;
5181         struct intel_encoder *encoder = NULL;
5182         int num_encoders = 0;
5183         int i;
5184
5185         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5186                 if (connector_state->crtc != &crtc->base)
5187                         continue;
5188
5189                 encoder = to_intel_encoder(connector_state->best_encoder);
5190                 num_encoders++;
5191         }
5192
5193         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5194              num_encoders, pipe_name(crtc->pipe));
5195
5196         return encoder;
5197 }
5198
5199 /*
5200  * Enable PCH resources required for PCH ports:
5201  *   - PCH PLLs
5202  *   - FDI training & RX/TX
5203  *   - update transcoder timings
5204  *   - DP transcoding bits
5205  *   - transcoder
5206  */
5207 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5208                                 const struct intel_crtc_state *crtc_state)
5209 {
5210         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5211         struct drm_device *dev = crtc->base.dev;
5212         struct drm_i915_private *dev_priv = to_i915(dev);
5213         int pipe = crtc->pipe;
5214         u32 temp;
5215
5216         assert_pch_transcoder_disabled(dev_priv, pipe);
5217
5218         if (IS_IVYBRIDGE(dev_priv))
5219                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5220
5221         /* Write the TU size bits before fdi link training, so that error
5222          * detection works. */
5223         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5224                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5225
5226         /* For PCH output, training FDI link */
5227         dev_priv->display.fdi_link_train(crtc, crtc_state);
5228
5229         /* We need to program the right clock selection before writing the pixel
5230          * mutliplier into the DPLL. */
5231         if (HAS_PCH_CPT(dev_priv)) {
5232                 u32 sel;
5233
5234                 temp = I915_READ(PCH_DPLL_SEL);
5235                 temp |= TRANS_DPLL_ENABLE(pipe);
5236                 sel = TRANS_DPLLB_SEL(pipe);
5237                 if (crtc_state->shared_dpll ==
5238                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5239                         temp |= sel;
5240                 else
5241                         temp &= ~sel;
5242                 I915_WRITE(PCH_DPLL_SEL, temp);
5243         }
5244
5245         /* XXX: pch pll's can be enabled any time before we enable the PCH
5246          * transcoder, and we actually should do this to not upset any PCH
5247          * transcoder that already use the clock when we share it.
5248          *
5249          * Note that enable_shared_dpll tries to do the right thing, but
5250          * get_shared_dpll unconditionally resets the pll - we need that to have
5251          * the right LVDS enable sequence. */
5252         intel_enable_shared_dpll(crtc_state);
5253
5254         /* set transcoder timing, panel must allow it */
5255         assert_panel_unlocked(dev_priv, pipe);
5256         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5257
5258         intel_fdi_normal_train(crtc);
5259
5260         /* For PCH DP, enable TRANS_DP_CTL */
5261         if (HAS_PCH_CPT(dev_priv) &&
5262             intel_crtc_has_dp_encoder(crtc_state)) {
5263                 const struct drm_display_mode *adjusted_mode =
5264                         &crtc_state->base.adjusted_mode;
5265                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5266                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5267                 enum port port;
5268
5269                 temp = I915_READ(reg);
5270                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5271                           TRANS_DP_SYNC_MASK |
5272                           TRANS_DP_BPC_MASK);
5273                 temp |= TRANS_DP_OUTPUT_ENABLE;
5274                 temp |= bpc << 9; /* same format but at 11:9 */
5275
5276                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5277                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5278                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5279                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5280
5281                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5282                 WARN_ON(port < PORT_B || port > PORT_D);
5283                 temp |= TRANS_DP_PORT_SEL(port);
5284
5285                 I915_WRITE(reg, temp);
5286         }
5287
5288         ironlake_enable_pch_transcoder(crtc_state);
5289 }
5290
5291 static void lpt_pch_enable(const struct intel_atomic_state *state,
5292                            const struct intel_crtc_state *crtc_state)
5293 {
5294         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5296         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5297
5298         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5299
5300         lpt_program_iclkip(crtc_state);
5301
5302         /* Set transcoder timing. */
5303         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5304
5305         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5306 }
5307
5308 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
5309 {
5310         struct drm_i915_private *dev_priv = to_i915(dev);
5311         i915_reg_t dslreg = PIPEDSL(pipe);
5312         u32 temp;
5313
5314         temp = I915_READ(dslreg);
5315         udelay(500);
5316         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5317                 if (wait_for(I915_READ(dslreg) != temp, 5))
5318                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5319         }
5320 }
5321
5322 /*
5323  * The hardware phase 0.0 refers to the center of the pixel.
5324  * We want to start from the top/left edge which is phase
5325  * -0.5. That matches how the hardware calculates the scaling
5326  * factors (from top-left of the first pixel to bottom-right
5327  * of the last pixel, as opposed to the pixel centers).
5328  *
5329  * For 4:2:0 subsampled chroma planes we obviously have to
5330  * adjust that so that the chroma sample position lands in
5331  * the right spot.
5332  *
5333  * Note that for packed YCbCr 4:2:2 formats there is no way to
5334  * control chroma siting. The hardware simply replicates the
5335  * chroma samples for both of the luma samples, and thus we don't
5336  * actually get the expected MPEG2 chroma siting convention :(
5337  * The same behaviour is observed on pre-SKL platforms as well.
5338  *
5339  * Theory behind the formula (note that we ignore sub-pixel
5340  * source coordinates):
5341  * s = source sample position
5342  * d = destination sample position
5343  *
5344  * Downscaling 4:1:
5345  * -0.5
5346  * | 0.0
5347  * | |     1.5 (initial phase)
5348  * | |     |
5349  * v v     v
5350  * | s | s | s | s |
5351  * |       d       |
5352  *
5353  * Upscaling 1:4:
5354  * -0.5
5355  * | -0.375 (initial phase)
5356  * | |     0.0
5357  * | |     |
5358  * v v     v
5359  * |       s       |
5360  * | d | d | d | d |
5361  */
5362 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5363 {
5364         int phase = -0x8000;
5365         u16 trip = 0;
5366
5367         if (chroma_cosited)
5368                 phase += (sub - 1) * 0x8000 / sub;
5369
5370         phase += scale / (2 * sub);
5371
5372         /*
5373          * Hardware initial phase limited to [-0.5:1.5].
5374          * Since the max hardware scale factor is 3.0, we
5375          * should never actually excdeed 1.0 here.
5376          */
5377         WARN_ON(phase < -0x8000 || phase > 0x18000);
5378
5379         if (phase < 0)
5380                 phase = 0x10000 + phase;
5381         else
5382                 trip = PS_PHASE_TRIP;
5383
5384         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5385 }
5386
5387 #define SKL_MIN_SRC_W 8
5388 #define SKL_MAX_SRC_W 4096
5389 #define SKL_MIN_SRC_H 8
5390 #define SKL_MAX_SRC_H 4096
5391 #define SKL_MIN_DST_W 8
5392 #define SKL_MAX_DST_W 4096
5393 #define SKL_MIN_DST_H 8
5394 #define SKL_MAX_DST_H 4096
5395 #define ICL_MAX_SRC_W 5120
5396 #define ICL_MAX_SRC_H 4096
5397 #define ICL_MAX_DST_W 5120
5398 #define ICL_MAX_DST_H 4096
5399 #define SKL_MIN_YUV_420_SRC_W 16
5400 #define SKL_MIN_YUV_420_SRC_H 16
5401
5402 static int
5403 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5404                   unsigned int scaler_user, int *scaler_id,
5405                   int src_w, int src_h, int dst_w, int dst_h,
5406                   const struct drm_format_info *format, bool need_scaler)
5407 {
5408         struct intel_crtc_scaler_state *scaler_state =
5409                 &crtc_state->scaler_state;
5410         struct intel_crtc *intel_crtc =
5411                 to_intel_crtc(crtc_state->base.crtc);
5412         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5413         const struct drm_display_mode *adjusted_mode =
5414                 &crtc_state->base.adjusted_mode;
5415
5416         /*
5417          * Src coordinates are already rotated by 270 degrees for
5418          * the 90/270 degree plane rotation cases (to match the
5419          * GTT mapping), hence no need to account for rotation here.
5420          */
5421         if (src_w != dst_w || src_h != dst_h)
5422                 need_scaler = true;
5423
5424         /*
5425          * Scaling/fitting not supported in IF-ID mode in GEN9+
5426          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5427          * Once NV12 is enabled, handle it here while allocating scaler
5428          * for NV12.
5429          */
5430         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
5431             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5432                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5433                 return -EINVAL;
5434         }
5435
5436         /*
5437          * if plane is being disabled or scaler is no more required or force detach
5438          *  - free scaler binded to this plane/crtc
5439          *  - in order to do this, update crtc->scaler_usage
5440          *
5441          * Here scaler state in crtc_state is set free so that
5442          * scaler can be assigned to other user. Actual register
5443          * update to free the scaler is done in plane/panel-fit programming.
5444          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5445          */
5446         if (force_detach || !need_scaler) {
5447                 if (*scaler_id >= 0) {
5448                         scaler_state->scaler_users &= ~(1 << scaler_user);
5449                         scaler_state->scalers[*scaler_id].in_use = 0;
5450
5451                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5452                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5453                                 intel_crtc->pipe, scaler_user, *scaler_id,
5454                                 scaler_state->scaler_users);
5455                         *scaler_id = -1;
5456                 }
5457                 return 0;
5458         }
5459
5460         if (format && is_planar_yuv_format(format->format) &&
5461             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5462                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5463                 return -EINVAL;
5464         }
5465
5466         /* range checks */
5467         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5468             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5469             (INTEL_GEN(dev_priv) >= 11 &&
5470              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5471               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5472             (INTEL_GEN(dev_priv) < 11 &&
5473              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5474               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5475                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5476                         "size is out of scaler range\n",
5477                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5478                 return -EINVAL;
5479         }
5480
5481         /* mark this plane as a scaler user in crtc_state */
5482         scaler_state->scaler_users |= (1 << scaler_user);
5483         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5484                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5485                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5486                 scaler_state->scaler_users);
5487
5488         return 0;
5489 }
5490
5491 /**
5492  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5493  *
5494  * @state: crtc's scaler state
5495  *
5496  * Return
5497  *     0 - scaler_usage updated successfully
5498  *    error - requested scaling cannot be supported or other error condition
5499  */
5500 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5501 {
5502         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
5503         bool need_scaler = false;
5504
5505         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5506                 need_scaler = true;
5507
5508         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
5509                                  &state->scaler_state.scaler_id,
5510                                  state->pipe_src_w, state->pipe_src_h,
5511                                  adjusted_mode->crtc_hdisplay,
5512                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
5513 }
5514
5515 /**
5516  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5517  * @crtc_state: crtc's scaler state
5518  * @plane_state: atomic plane state to update
5519  *
5520  * Return
5521  *     0 - scaler_usage updated successfully
5522  *    error - requested scaling cannot be supported or other error condition
5523  */
5524 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5525                                    struct intel_plane_state *plane_state)
5526 {
5527         struct intel_plane *intel_plane =
5528                 to_intel_plane(plane_state->base.plane);
5529         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5530         struct drm_framebuffer *fb = plane_state->base.fb;
5531         int ret;
5532         bool force_detach = !fb || !plane_state->base.visible;
5533         bool need_scaler = false;
5534
5535         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5536         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5537             fb && is_planar_yuv_format(fb->format->format))
5538                 need_scaler = true;
5539
5540         ret = skl_update_scaler(crtc_state, force_detach,
5541                                 drm_plane_index(&intel_plane->base),
5542                                 &plane_state->scaler_id,
5543                                 drm_rect_width(&plane_state->base.src) >> 16,
5544                                 drm_rect_height(&plane_state->base.src) >> 16,
5545                                 drm_rect_width(&plane_state->base.dst),
5546                                 drm_rect_height(&plane_state->base.dst),
5547                                 fb ? fb->format : NULL, need_scaler);
5548
5549         if (ret || plane_state->scaler_id < 0)
5550                 return ret;
5551
5552         /* check colorkey */
5553         if (plane_state->ckey.flags) {
5554                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5555                               intel_plane->base.base.id,
5556                               intel_plane->base.name);
5557                 return -EINVAL;
5558         }
5559
5560         /* Check src format */
5561         switch (fb->format->format) {
5562         case DRM_FORMAT_RGB565:
5563         case DRM_FORMAT_XBGR8888:
5564         case DRM_FORMAT_XRGB8888:
5565         case DRM_FORMAT_ABGR8888:
5566         case DRM_FORMAT_ARGB8888:
5567         case DRM_FORMAT_XRGB2101010:
5568         case DRM_FORMAT_XBGR2101010:
5569         case DRM_FORMAT_XBGR16161616F:
5570         case DRM_FORMAT_ABGR16161616F:
5571         case DRM_FORMAT_XRGB16161616F:
5572         case DRM_FORMAT_ARGB16161616F:
5573         case DRM_FORMAT_YUYV:
5574         case DRM_FORMAT_YVYU:
5575         case DRM_FORMAT_UYVY:
5576         case DRM_FORMAT_VYUY:
5577         case DRM_FORMAT_NV12:
5578         case DRM_FORMAT_P010:
5579         case DRM_FORMAT_P012:
5580         case DRM_FORMAT_P016:
5581         case DRM_FORMAT_Y210:
5582         case DRM_FORMAT_Y212:
5583         case DRM_FORMAT_Y216:
5584         case DRM_FORMAT_XVYU2101010:
5585         case DRM_FORMAT_XVYU12_16161616:
5586         case DRM_FORMAT_XVYU16161616:
5587                 break;
5588         default:
5589                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5590                               intel_plane->base.base.id, intel_plane->base.name,
5591                               fb->base.id, fb->format->format);
5592                 return -EINVAL;
5593         }
5594
5595         return 0;
5596 }
5597
5598 static void skylake_scaler_disable(struct intel_crtc *crtc)
5599 {
5600         int i;
5601
5602         for (i = 0; i < crtc->num_scalers; i++)
5603                 skl_detach_scaler(crtc, i);
5604 }
5605
5606 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5607 {
5608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5609         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5610         enum pipe pipe = crtc->pipe;
5611         const struct intel_crtc_scaler_state *scaler_state =
5612                 &crtc_state->scaler_state;
5613
5614         if (crtc_state->pch_pfit.enabled) {
5615                 u16 uv_rgb_hphase, uv_rgb_vphase;
5616                 int pfit_w, pfit_h, hscale, vscale;
5617                 int id;
5618
5619                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5620                         return;
5621
5622                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5623                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5624
5625                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5626                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5627
5628                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5629                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5630
5631                 id = scaler_state->scaler_id;
5632                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5633                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5634                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5635                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5636                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5637                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5638                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5639                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5640         }
5641 }
5642
5643 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5644 {
5645         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5646         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5647         int pipe = crtc->pipe;
5648
5649         if (crtc_state->pch_pfit.enabled) {
5650                 /* Force use of hard-coded filter coefficients
5651                  * as some pre-programmed values are broken,
5652                  * e.g. x201.
5653                  */
5654                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5655                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5656                                                  PF_PIPE_SEL_IVB(pipe));
5657                 else
5658                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5659                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5660                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5661         }
5662 }
5663
5664 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5665 {
5666         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5667         struct drm_device *dev = crtc->base.dev;
5668         struct drm_i915_private *dev_priv = to_i915(dev);
5669
5670         if (!crtc_state->ips_enabled)
5671                 return;
5672
5673         /*
5674          * We can only enable IPS after we enable a plane and wait for a vblank
5675          * This function is called from post_plane_update, which is run after
5676          * a vblank wait.
5677          */
5678         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5679
5680         if (IS_BROADWELL(dev_priv)) {
5681                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5682                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5683                 /* Quoting Art Runyan: "its not safe to expect any particular
5684                  * value in IPS_CTL bit 31 after enabling IPS through the
5685                  * mailbox." Moreover, the mailbox may return a bogus state,
5686                  * so we need to just enable it and continue on.
5687                  */
5688         } else {
5689                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5690                 /* The bit only becomes 1 in the next vblank, so this wait here
5691                  * is essentially intel_wait_for_vblank. If we don't have this
5692                  * and don't wait for vblanks until the end of crtc_enable, then
5693                  * the HW state readout code will complain that the expected
5694                  * IPS_CTL value is not the one we read. */
5695                 if (intel_wait_for_register(&dev_priv->uncore,
5696                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5697                                             50))
5698                         DRM_ERROR("Timed out waiting for IPS enable\n");
5699         }
5700 }
5701
5702 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5703 {
5704         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5705         struct drm_device *dev = crtc->base.dev;
5706         struct drm_i915_private *dev_priv = to_i915(dev);
5707
5708         if (!crtc_state->ips_enabled)
5709                 return;
5710
5711         if (IS_BROADWELL(dev_priv)) {
5712                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5713                 /*
5714                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5715                  * 42ms timeout value leads to occasional timeouts so use 100ms
5716                  * instead.
5717                  */
5718                 if (intel_wait_for_register(&dev_priv->uncore,
5719                                             IPS_CTL, IPS_ENABLE, 0,
5720                                             100))
5721                         DRM_ERROR("Timed out waiting for IPS disable\n");
5722         } else {
5723                 I915_WRITE(IPS_CTL, 0);
5724                 POSTING_READ(IPS_CTL);
5725         }
5726
5727         /* We need to wait for a vblank before we can disable the plane. */
5728         intel_wait_for_vblank(dev_priv, crtc->pipe);
5729 }
5730
5731 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5732 {
5733         if (intel_crtc->overlay) {
5734                 struct drm_device *dev = intel_crtc->base.dev;
5735
5736                 mutex_lock(&dev->struct_mutex);
5737                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5738                 mutex_unlock(&dev->struct_mutex);
5739         }
5740
5741         /* Let userspace switch the overlay on again. In most cases userspace
5742          * has to recompute where to put it anyway.
5743          */
5744 }
5745
5746 /**
5747  * intel_post_enable_primary - Perform operations after enabling primary plane
5748  * @crtc: the CRTC whose primary plane was just enabled
5749  * @new_crtc_state: the enabling state
5750  *
5751  * Performs potentially sleeping operations that must be done after the primary
5752  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5753  * called due to an explicit primary plane update, or due to an implicit
5754  * re-enable that is caused when a sprite plane is updated to no longer
5755  * completely hide the primary plane.
5756  */
5757 static void
5758 intel_post_enable_primary(struct drm_crtc *crtc,
5759                           const struct intel_crtc_state *new_crtc_state)
5760 {
5761         struct drm_device *dev = crtc->dev;
5762         struct drm_i915_private *dev_priv = to_i915(dev);
5763         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5764         int pipe = intel_crtc->pipe;
5765
5766         /*
5767          * Gen2 reports pipe underruns whenever all planes are disabled.
5768          * So don't enable underrun reporting before at least some planes
5769          * are enabled.
5770          * FIXME: Need to fix the logic to work when we turn off all planes
5771          * but leave the pipe running.
5772          */
5773         if (IS_GEN(dev_priv, 2))
5774                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5775
5776         /* Underruns don't always raise interrupts, so check manually. */
5777         intel_check_cpu_fifo_underruns(dev_priv);
5778         intel_check_pch_fifo_underruns(dev_priv);
5779 }
5780
5781 /* FIXME get rid of this and use pre_plane_update */
5782 static void
5783 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5784 {
5785         struct drm_device *dev = crtc->dev;
5786         struct drm_i915_private *dev_priv = to_i915(dev);
5787         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5788         int pipe = intel_crtc->pipe;
5789
5790         /*
5791          * Gen2 reports pipe underruns whenever all planes are disabled.
5792          * So disable underrun reporting before all the planes get disabled.
5793          */
5794         if (IS_GEN(dev_priv, 2))
5795                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5796
5797         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5798
5799         /*
5800          * Vblank time updates from the shadow to live plane control register
5801          * are blocked if the memory self-refresh mode is active at that
5802          * moment. So to make sure the plane gets truly disabled, disable
5803          * first the self-refresh mode. The self-refresh enable bit in turn
5804          * will be checked/applied by the HW only at the next frame start
5805          * event which is after the vblank start event, so we need to have a
5806          * wait-for-vblank between disabling the plane and the pipe.
5807          */
5808         if (HAS_GMCH(dev_priv) &&
5809             intel_set_memory_cxsr(dev_priv, false))
5810                 intel_wait_for_vblank(dev_priv, pipe);
5811 }
5812
5813 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5814                                        const struct intel_crtc_state *new_crtc_state)
5815 {
5816         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5817         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5818
5819         if (!old_crtc_state->ips_enabled)
5820                 return false;
5821
5822         if (needs_modeset(new_crtc_state))
5823                 return true;
5824
5825         /*
5826          * Workaround : Do not read or write the pipe palette/gamma data while
5827          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5828          *
5829          * Disable IPS before we program the LUT.
5830          */
5831         if (IS_HASWELL(dev_priv) &&
5832             (new_crtc_state->base.color_mgmt_changed ||
5833              new_crtc_state->update_pipe) &&
5834             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5835                 return true;
5836
5837         return !new_crtc_state->ips_enabled;
5838 }
5839
5840 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5841                                        const struct intel_crtc_state *new_crtc_state)
5842 {
5843         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5844         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5845
5846         if (!new_crtc_state->ips_enabled)
5847                 return false;
5848
5849         if (needs_modeset(new_crtc_state))
5850                 return true;
5851
5852         /*
5853          * Workaround : Do not read or write the pipe palette/gamma data while
5854          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5855          *
5856          * Re-enable IPS after the LUT has been programmed.
5857          */
5858         if (IS_HASWELL(dev_priv) &&
5859             (new_crtc_state->base.color_mgmt_changed ||
5860              new_crtc_state->update_pipe) &&
5861             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5862                 return true;
5863
5864         /*
5865          * We can't read out IPS on broadwell, assume the worst and
5866          * forcibly enable IPS on the first fastset.
5867          */
5868         if (new_crtc_state->update_pipe &&
5869             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5870                 return true;
5871
5872         return !old_crtc_state->ips_enabled;
5873 }
5874
5875 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5876                           const struct intel_crtc_state *crtc_state)
5877 {
5878         if (!crtc_state->nv12_planes)
5879                 return false;
5880
5881         /* WA Display #0827: Gen9:all */
5882         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
5883                 return true;
5884
5885         return false;
5886 }
5887
5888 static bool needs_scalerclk_wa(struct drm_i915_private *dev_priv,
5889                                const struct intel_crtc_state *crtc_state)
5890 {
5891         /* Wa_2006604312:icl */
5892         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
5893                 return true;
5894
5895         return false;
5896 }
5897
5898 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5899 {
5900         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5901         struct drm_device *dev = crtc->base.dev;
5902         struct drm_i915_private *dev_priv = to_i915(dev);
5903         struct drm_atomic_state *state = old_crtc_state->base.state;
5904         struct intel_crtc_state *pipe_config =
5905                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(state),
5906                                                 crtc);
5907         struct drm_plane *primary = crtc->base.primary;
5908         struct drm_plane_state *old_primary_state =
5909                 drm_atomic_get_old_plane_state(state, primary);
5910
5911         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5912
5913         if (pipe_config->update_wm_post && pipe_config->base.active)
5914                 intel_update_watermarks(crtc);
5915
5916         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5917                 hsw_enable_ips(pipe_config);
5918
5919         if (old_primary_state) {
5920                 struct drm_plane_state *new_primary_state =
5921                         drm_atomic_get_new_plane_state(state, primary);
5922
5923                 intel_fbc_post_update(crtc);
5924
5925                 if (new_primary_state->visible &&
5926                     (needs_modeset(pipe_config) ||
5927                      !old_primary_state->visible))
5928                         intel_post_enable_primary(&crtc->base, pipe_config);
5929         }
5930
5931         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5932             !needs_nv12_wa(dev_priv, pipe_config))
5933                 skl_wa_827(dev_priv, crtc->pipe, false);
5934
5935         if (needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5936             !needs_scalerclk_wa(dev_priv, pipe_config))
5937                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, false);
5938 }
5939
5940 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5941                                    struct intel_crtc_state *pipe_config)
5942 {
5943         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5944         struct drm_device *dev = crtc->base.dev;
5945         struct drm_i915_private *dev_priv = to_i915(dev);
5946         struct drm_atomic_state *state = old_crtc_state->base.state;
5947         struct drm_plane *primary = crtc->base.primary;
5948         struct drm_plane_state *old_primary_state =
5949                 drm_atomic_get_old_plane_state(state, primary);
5950         bool modeset = needs_modeset(pipe_config);
5951         struct intel_atomic_state *intel_state =
5952                 to_intel_atomic_state(state);
5953
5954         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5955                 hsw_disable_ips(old_crtc_state);
5956
5957         if (old_primary_state) {
5958                 struct intel_plane_state *new_primary_state =
5959                         intel_atomic_get_new_plane_state(intel_state,
5960                                                          to_intel_plane(primary));
5961
5962                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5963                 /*
5964                  * Gen2 reports pipe underruns whenever all planes are disabled.
5965                  * So disable underrun reporting before all the planes get disabled.
5966                  */
5967                 if (IS_GEN(dev_priv, 2) && old_primary_state->visible &&
5968                     (modeset || !new_primary_state->base.visible))
5969                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5970         }
5971
5972         /* Display WA 827 */
5973         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5974             needs_nv12_wa(dev_priv, pipe_config))
5975                 skl_wa_827(dev_priv, crtc->pipe, true);
5976
5977         /* Wa_2006604312:icl */
5978         if (!needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5979             needs_scalerclk_wa(dev_priv, pipe_config))
5980                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, true);
5981
5982         /*
5983          * Vblank time updates from the shadow to live plane control register
5984          * are blocked if the memory self-refresh mode is active at that
5985          * moment. So to make sure the plane gets truly disabled, disable
5986          * first the self-refresh mode. The self-refresh enable bit in turn
5987          * will be checked/applied by the HW only at the next frame start
5988          * event which is after the vblank start event, so we need to have a
5989          * wait-for-vblank between disabling the plane and the pipe.
5990          */
5991         if (HAS_GMCH(dev_priv) && old_crtc_state->base.active &&
5992             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5993                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5994
5995         /*
5996          * IVB workaround: must disable low power watermarks for at least
5997          * one frame before enabling scaling.  LP watermarks can be re-enabled
5998          * when scaling is disabled.
5999          *
6000          * WaCxSRDisabledForSpriteScaling:ivb
6001          */
6002         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
6003             old_crtc_state->base.active)
6004                 intel_wait_for_vblank(dev_priv, crtc->pipe);
6005
6006         /*
6007          * If we're doing a modeset, we're done.  No need to do any pre-vblank
6008          * watermark programming here.
6009          */
6010         if (needs_modeset(pipe_config))
6011                 return;
6012
6013         /*
6014          * For platforms that support atomic watermarks, program the
6015          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6016          * will be the intermediate values that are safe for both pre- and
6017          * post- vblank; when vblank happens, the 'active' values will be set
6018          * to the final 'target' values and we'll do this again to get the
6019          * optimal watermarks.  For gen9+ platforms, the values we program here
6020          * will be the final target values which will get automatically latched
6021          * at vblank time; no further programming will be necessary.
6022          *
6023          * If a platform hasn't been transitioned to atomic watermarks yet,
6024          * we'll continue to update watermarks the old way, if flags tell
6025          * us to.
6026          */
6027         if (dev_priv->display.initial_watermarks != NULL)
6028                 dev_priv->display.initial_watermarks(intel_state,
6029                                                      pipe_config);
6030         else if (pipe_config->update_wm_pre)
6031                 intel_update_watermarks(crtc);
6032 }
6033
6034 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6035                                       struct intel_crtc *crtc)
6036 {
6037         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6038         const struct intel_crtc_state *new_crtc_state =
6039                 intel_atomic_get_new_crtc_state(state, crtc);
6040         unsigned int update_mask = new_crtc_state->update_planes;
6041         const struct intel_plane_state *old_plane_state;
6042         struct intel_plane *plane;
6043         unsigned fb_bits = 0;
6044         int i;
6045
6046         intel_crtc_dpms_overlay_disable(crtc);
6047
6048         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6049                 if (crtc->pipe != plane->pipe ||
6050                     !(update_mask & BIT(plane->id)))
6051                         continue;
6052
6053                 intel_disable_plane(plane, new_crtc_state);
6054
6055                 if (old_plane_state->base.visible)
6056                         fb_bits |= plane->frontbuffer_bit;
6057         }
6058
6059         intel_frontbuffer_flip(dev_priv, fb_bits);
6060 }
6061
6062 /*
6063  * intel_connector_primary_encoder - get the primary encoder for a connector
6064  * @connector: connector for which to return the encoder
6065  *
6066  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6067  * all connectors to their encoder, except for DP-MST connectors which have
6068  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6069  * pointed to by as many DP-MST connectors as there are pipes.
6070  */
6071 static struct intel_encoder *
6072 intel_connector_primary_encoder(struct intel_connector *connector)
6073 {
6074         struct intel_encoder *encoder;
6075
6076         if (connector->mst_port)
6077                 return &dp_to_dig_port(connector->mst_port)->base;
6078
6079         encoder = intel_attached_encoder(&connector->base);
6080         WARN_ON(!encoder);
6081
6082         return encoder;
6083 }
6084
6085 static bool
6086 intel_connector_needs_modeset(struct intel_atomic_state *state,
6087                               const struct drm_connector_state *old_conn_state,
6088                               const struct drm_connector_state *new_conn_state)
6089 {
6090         struct intel_crtc *old_crtc = old_conn_state->crtc ?
6091                                       to_intel_crtc(old_conn_state->crtc) : NULL;
6092         struct intel_crtc *new_crtc = new_conn_state->crtc ?
6093                                       to_intel_crtc(new_conn_state->crtc) : NULL;
6094
6095         return new_crtc != old_crtc ||
6096                (new_crtc &&
6097                 needs_modeset(intel_atomic_get_new_crtc_state(state, new_crtc)));
6098 }
6099
6100 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6101 {
6102         struct drm_connector_state *old_conn_state;
6103         struct drm_connector_state *new_conn_state;
6104         struct drm_connector *conn;
6105         int i;
6106
6107         for_each_oldnew_connector_in_state(&state->base, conn,
6108                                            old_conn_state, new_conn_state, i) {
6109                 struct intel_encoder *encoder;
6110                 struct intel_crtc *crtc;
6111
6112                 if (!intel_connector_needs_modeset(state,
6113                                                    old_conn_state,
6114                                                    new_conn_state))
6115                         continue;
6116
6117                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6118                 if (!encoder->update_prepare)
6119                         continue;
6120
6121                 crtc = new_conn_state->crtc ?
6122                         to_intel_crtc(new_conn_state->crtc) : NULL;
6123                 encoder->update_prepare(state, encoder, crtc);
6124         }
6125 }
6126
6127 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6128 {
6129         struct drm_connector_state *old_conn_state;
6130         struct drm_connector_state *new_conn_state;
6131         struct drm_connector *conn;
6132         int i;
6133
6134         for_each_oldnew_connector_in_state(&state->base, conn,
6135                                            old_conn_state, new_conn_state, i) {
6136                 struct intel_encoder *encoder;
6137                 struct intel_crtc *crtc;
6138
6139                 if (!intel_connector_needs_modeset(state,
6140                                                    old_conn_state,
6141                                                    new_conn_state))
6142                         continue;
6143
6144                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6145                 if (!encoder->update_complete)
6146                         continue;
6147
6148                 crtc = new_conn_state->crtc ?
6149                         to_intel_crtc(new_conn_state->crtc) : NULL;
6150                 encoder->update_complete(state, encoder, crtc);
6151         }
6152 }
6153
6154 static void intel_encoders_pre_pll_enable(struct intel_crtc *crtc,
6155                                           struct intel_crtc_state *crtc_state,
6156                                           struct intel_atomic_state *state)
6157 {
6158         struct drm_connector_state *conn_state;
6159         struct drm_connector *conn;
6160         int i;
6161
6162         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6163                 struct intel_encoder *encoder =
6164                         to_intel_encoder(conn_state->best_encoder);
6165
6166                 if (conn_state->crtc != &crtc->base)
6167                         continue;
6168
6169                 if (encoder->pre_pll_enable)
6170                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6171         }
6172 }
6173
6174 static void intel_encoders_pre_enable(struct intel_crtc *crtc,
6175                                       struct intel_crtc_state *crtc_state,
6176                                       struct intel_atomic_state *state)
6177 {
6178         struct drm_connector_state *conn_state;
6179         struct drm_connector *conn;
6180         int i;
6181
6182         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6183                 struct intel_encoder *encoder =
6184                         to_intel_encoder(conn_state->best_encoder);
6185
6186                 if (conn_state->crtc != &crtc->base)
6187                         continue;
6188
6189                 if (encoder->pre_enable)
6190                         encoder->pre_enable(encoder, crtc_state, conn_state);
6191         }
6192 }
6193
6194 static void intel_encoders_enable(struct intel_crtc *crtc,
6195                                   struct intel_crtc_state *crtc_state,
6196                                   struct intel_atomic_state *state)
6197 {
6198         struct drm_connector_state *conn_state;
6199         struct drm_connector *conn;
6200         int i;
6201
6202         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6203                 struct intel_encoder *encoder =
6204                         to_intel_encoder(conn_state->best_encoder);
6205
6206                 if (conn_state->crtc != &crtc->base)
6207                         continue;
6208
6209                 if (encoder->enable)
6210                         encoder->enable(encoder, crtc_state, conn_state);
6211                 intel_opregion_notify_encoder(encoder, true);
6212         }
6213 }
6214
6215 static void intel_encoders_disable(struct intel_crtc *crtc,
6216                                    struct intel_crtc_state *old_crtc_state,
6217                                    struct intel_atomic_state *state)
6218 {
6219         struct drm_connector_state *old_conn_state;
6220         struct drm_connector *conn;
6221         int i;
6222
6223         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6224                 struct intel_encoder *encoder =
6225                         to_intel_encoder(old_conn_state->best_encoder);
6226
6227                 if (old_conn_state->crtc != &crtc->base)
6228                         continue;
6229
6230                 intel_opregion_notify_encoder(encoder, false);
6231                 if (encoder->disable)
6232                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6233         }
6234 }
6235
6236 static void intel_encoders_post_disable(struct intel_crtc *crtc,
6237                                         struct intel_crtc_state *old_crtc_state,
6238                                         struct intel_atomic_state *state)
6239 {
6240         struct drm_connector_state *old_conn_state;
6241         struct drm_connector *conn;
6242         int i;
6243
6244         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6245                 struct intel_encoder *encoder =
6246                         to_intel_encoder(old_conn_state->best_encoder);
6247
6248                 if (old_conn_state->crtc != &crtc->base)
6249                         continue;
6250
6251                 if (encoder->post_disable)
6252                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6253         }
6254 }
6255
6256 static void intel_encoders_post_pll_disable(struct intel_crtc *crtc,
6257                                             struct intel_crtc_state *old_crtc_state,
6258                                             struct intel_atomic_state *state)
6259 {
6260         struct drm_connector_state *old_conn_state;
6261         struct drm_connector *conn;
6262         int i;
6263
6264         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6265                 struct intel_encoder *encoder =
6266                         to_intel_encoder(old_conn_state->best_encoder);
6267
6268                 if (old_conn_state->crtc != &crtc->base)
6269                         continue;
6270
6271                 if (encoder->post_pll_disable)
6272                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6273         }
6274 }
6275
6276 static void intel_encoders_update_pipe(struct intel_crtc *crtc,
6277                                        struct intel_crtc_state *crtc_state,
6278                                        struct intel_atomic_state *state)
6279 {
6280         struct drm_connector_state *conn_state;
6281         struct drm_connector *conn;
6282         int i;
6283
6284         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6285                 struct intel_encoder *encoder =
6286                         to_intel_encoder(conn_state->best_encoder);
6287
6288                 if (conn_state->crtc != &crtc->base)
6289                         continue;
6290
6291                 if (encoder->update_pipe)
6292                         encoder->update_pipe(encoder, crtc_state, conn_state);
6293         }
6294 }
6295
6296 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6297 {
6298         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6299         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6300
6301         plane->disable_plane(plane, crtc_state);
6302 }
6303
6304 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
6305                                  struct intel_atomic_state *state)
6306 {
6307         struct drm_crtc *crtc = pipe_config->base.crtc;
6308         struct drm_device *dev = crtc->dev;
6309         struct drm_i915_private *dev_priv = to_i915(dev);
6310         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6311         int pipe = intel_crtc->pipe;
6312
6313         if (WARN_ON(intel_crtc->active))
6314                 return;
6315
6316         /*
6317          * Sometimes spurious CPU pipe underruns happen during FDI
6318          * training, at least with VGA+HDMI cloning. Suppress them.
6319          *
6320          * On ILK we get an occasional spurious CPU pipe underruns
6321          * between eDP port A enable and vdd enable. Also PCH port
6322          * enable seems to result in the occasional CPU pipe underrun.
6323          *
6324          * Spurious PCH underruns also occur during PCH enabling.
6325          */
6326         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6327         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6328
6329         if (pipe_config->has_pch_encoder)
6330                 intel_prepare_shared_dpll(pipe_config);
6331
6332         if (intel_crtc_has_dp_encoder(pipe_config))
6333                 intel_dp_set_m_n(pipe_config, M1_N1);
6334
6335         intel_set_pipe_timings(pipe_config);
6336         intel_set_pipe_src_size(pipe_config);
6337
6338         if (pipe_config->has_pch_encoder) {
6339                 intel_cpu_transcoder_set_m_n(pipe_config,
6340                                              &pipe_config->fdi_m_n, NULL);
6341         }
6342
6343         ironlake_set_pipeconf(pipe_config);
6344
6345         intel_crtc->active = true;
6346
6347         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6348
6349         if (pipe_config->has_pch_encoder) {
6350                 /* Note: FDI PLL enabling _must_ be done before we enable the
6351                  * cpu pipes, hence this is separate from all the other fdi/pch
6352                  * enabling. */
6353                 ironlake_fdi_pll_enable(pipe_config);
6354         } else {
6355                 assert_fdi_tx_disabled(dev_priv, pipe);
6356                 assert_fdi_rx_disabled(dev_priv, pipe);
6357         }
6358
6359         ironlake_pfit_enable(pipe_config);
6360
6361         /*
6362          * On ILK+ LUT must be loaded before the pipe is running but with
6363          * clocks enabled
6364          */
6365         intel_color_load_luts(pipe_config);
6366         intel_color_commit(pipe_config);
6367         /* update DSPCNTR to configure gamma for pipe bottom color */
6368         intel_disable_primary_plane(pipe_config);
6369
6370         if (dev_priv->display.initial_watermarks != NULL)
6371                 dev_priv->display.initial_watermarks(state, pipe_config);
6372         intel_enable_pipe(pipe_config);
6373
6374         if (pipe_config->has_pch_encoder)
6375                 ironlake_pch_enable(state, pipe_config);
6376
6377         assert_vblank_disabled(crtc);
6378         intel_crtc_vblank_on(pipe_config);
6379
6380         intel_encoders_enable(intel_crtc, pipe_config, state);
6381
6382         if (HAS_PCH_CPT(dev_priv))
6383                 cpt_verify_modeset(dev, intel_crtc->pipe);
6384
6385         /*
6386          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6387          * And a second vblank wait is needed at least on ILK with
6388          * some interlaced HDMI modes. Let's do the double wait always
6389          * in case there are more corner cases we don't know about.
6390          */
6391         if (pipe_config->has_pch_encoder) {
6392                 intel_wait_for_vblank(dev_priv, pipe);
6393                 intel_wait_for_vblank(dev_priv, pipe);
6394         }
6395         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6396         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6397 }
6398
6399 /* IPS only exists on ULT machines and is tied to pipe A. */
6400 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6401 {
6402         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6403 }
6404
6405 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6406                                             enum pipe pipe, bool apply)
6407 {
6408         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6409         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6410
6411         if (apply)
6412                 val |= mask;
6413         else
6414                 val &= ~mask;
6415
6416         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6417 }
6418
6419 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6420 {
6421         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6422         enum pipe pipe = crtc->pipe;
6423         u32 val;
6424
6425         val = MBUS_DBOX_A_CREDIT(2);
6426         val |= MBUS_DBOX_BW_CREDIT(1);
6427         val |= MBUS_DBOX_B_CREDIT(8);
6428
6429         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6430 }
6431
6432 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
6433                                 struct intel_atomic_state *state)
6434 {
6435         struct drm_crtc *crtc = pipe_config->base.crtc;
6436         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6437         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6438         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
6439         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
6440         bool psl_clkgate_wa;
6441
6442         if (WARN_ON(intel_crtc->active))
6443                 return;
6444
6445         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6446
6447         if (pipe_config->shared_dpll)
6448                 intel_enable_shared_dpll(pipe_config);
6449
6450         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6451
6452         if (intel_crtc_has_dp_encoder(pipe_config))
6453                 intel_dp_set_m_n(pipe_config, M1_N1);
6454
6455         if (!transcoder_is_dsi(cpu_transcoder))
6456                 intel_set_pipe_timings(pipe_config);
6457
6458         intel_set_pipe_src_size(pipe_config);
6459
6460         if (cpu_transcoder != TRANSCODER_EDP &&
6461             !transcoder_is_dsi(cpu_transcoder)) {
6462                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6463                            pipe_config->pixel_multiplier - 1);
6464         }
6465
6466         if (pipe_config->has_pch_encoder) {
6467                 intel_cpu_transcoder_set_m_n(pipe_config,
6468                                              &pipe_config->fdi_m_n, NULL);
6469         }
6470
6471         if (!transcoder_is_dsi(cpu_transcoder))
6472                 haswell_set_pipeconf(pipe_config);
6473
6474         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6475                 bdw_set_pipemisc(pipe_config);
6476
6477         intel_crtc->active = true;
6478
6479         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6480         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6481                          pipe_config->pch_pfit.enabled;
6482         if (psl_clkgate_wa)
6483                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6484
6485         if (INTEL_GEN(dev_priv) >= 9)
6486                 skylake_pfit_enable(pipe_config);
6487         else
6488                 ironlake_pfit_enable(pipe_config);
6489
6490         /*
6491          * On ILK+ LUT must be loaded before the pipe is running but with
6492          * clocks enabled
6493          */
6494         intel_color_load_luts(pipe_config);
6495         intel_color_commit(pipe_config);
6496         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6497         if (INTEL_GEN(dev_priv) < 9)
6498                 intel_disable_primary_plane(pipe_config);
6499
6500         if (INTEL_GEN(dev_priv) >= 11)
6501                 icl_set_pipe_chicken(intel_crtc);
6502
6503         intel_ddi_set_pipe_settings(pipe_config);
6504         if (!transcoder_is_dsi(cpu_transcoder))
6505                 intel_ddi_enable_transcoder_func(pipe_config);
6506
6507         if (dev_priv->display.initial_watermarks != NULL)
6508                 dev_priv->display.initial_watermarks(state, pipe_config);
6509
6510         if (INTEL_GEN(dev_priv) >= 11)
6511                 icl_pipe_mbus_enable(intel_crtc);
6512
6513         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6514         if (!transcoder_is_dsi(cpu_transcoder))
6515                 intel_enable_pipe(pipe_config);
6516
6517         if (pipe_config->has_pch_encoder)
6518                 lpt_pch_enable(state, pipe_config);
6519
6520         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
6521                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
6522
6523         assert_vblank_disabled(crtc);
6524         intel_crtc_vblank_on(pipe_config);
6525
6526         intel_encoders_enable(intel_crtc, pipe_config, state);
6527
6528         if (psl_clkgate_wa) {
6529                 intel_wait_for_vblank(dev_priv, pipe);
6530                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6531         }
6532
6533         /* If we change the relative order between pipe/planes enabling, we need
6534          * to change the workaround. */
6535         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
6536         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6537                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6538                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6539         }
6540 }
6541
6542 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6543 {
6544         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6545         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6546         enum pipe pipe = crtc->pipe;
6547
6548         /* To avoid upsetting the power well on haswell only disable the pfit if
6549          * it's in use. The hw state code will make sure we get this right. */
6550         if (old_crtc_state->pch_pfit.enabled) {
6551                 I915_WRITE(PF_CTL(pipe), 0);
6552                 I915_WRITE(PF_WIN_POS(pipe), 0);
6553                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6554         }
6555 }
6556
6557 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
6558                                   struct intel_atomic_state *state)
6559 {
6560         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6561         struct drm_device *dev = crtc->dev;
6562         struct drm_i915_private *dev_priv = to_i915(dev);
6563         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6564         int pipe = intel_crtc->pipe;
6565
6566         /*
6567          * Sometimes spurious CPU pipe underruns happen when the
6568          * pipe is already disabled, but FDI RX/TX is still enabled.
6569          * Happens at least with VGA+HDMI cloning. Suppress them.
6570          */
6571         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6572         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6573
6574         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6575
6576         drm_crtc_vblank_off(crtc);
6577         assert_vblank_disabled(crtc);
6578
6579         intel_disable_pipe(old_crtc_state);
6580
6581         ironlake_pfit_disable(old_crtc_state);
6582
6583         if (old_crtc_state->has_pch_encoder)
6584                 ironlake_fdi_disable(crtc);
6585
6586         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6587
6588         if (old_crtc_state->has_pch_encoder) {
6589                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6590
6591                 if (HAS_PCH_CPT(dev_priv)) {
6592                         i915_reg_t reg;
6593                         u32 temp;
6594
6595                         /* disable TRANS_DP_CTL */
6596                         reg = TRANS_DP_CTL(pipe);
6597                         temp = I915_READ(reg);
6598                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6599                                   TRANS_DP_PORT_SEL_MASK);
6600                         temp |= TRANS_DP_PORT_SEL_NONE;
6601                         I915_WRITE(reg, temp);
6602
6603                         /* disable DPLL_SEL */
6604                         temp = I915_READ(PCH_DPLL_SEL);
6605                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6606                         I915_WRITE(PCH_DPLL_SEL, temp);
6607                 }
6608
6609                 ironlake_fdi_pll_disable(intel_crtc);
6610         }
6611
6612         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6613         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6614 }
6615
6616 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
6617                                  struct intel_atomic_state *state)
6618 {
6619         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6620         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6621         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6622         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
6623
6624         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6625
6626         drm_crtc_vblank_off(crtc);
6627         assert_vblank_disabled(crtc);
6628
6629         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6630         if (!transcoder_is_dsi(cpu_transcoder))
6631                 intel_disable_pipe(old_crtc_state);
6632
6633         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
6634                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
6635
6636         if (!transcoder_is_dsi(cpu_transcoder))
6637                 intel_ddi_disable_transcoder_func(old_crtc_state);
6638
6639         intel_dsc_disable(old_crtc_state);
6640
6641         if (INTEL_GEN(dev_priv) >= 9)
6642                 skylake_scaler_disable(intel_crtc);
6643         else
6644                 ironlake_pfit_disable(old_crtc_state);
6645
6646         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6647
6648         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
6649 }
6650
6651 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6652 {
6653         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6654         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6655
6656         if (!crtc_state->gmch_pfit.control)
6657                 return;
6658
6659         /*
6660          * The panel fitter should only be adjusted whilst the pipe is disabled,
6661          * according to register description and PRM.
6662          */
6663         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6664         assert_pipe_disabled(dev_priv, crtc->pipe);
6665
6666         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6667         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6668
6669         /* Border color in case we don't scale up to the full screen. Black by
6670          * default, change to something else for debugging. */
6671         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6672 }
6673
6674 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6675 {
6676         if (phy == PHY_NONE)
6677                 return false;
6678
6679         if (IS_ELKHARTLAKE(dev_priv))
6680                 return phy <= PHY_C;
6681
6682         if (INTEL_GEN(dev_priv) >= 11)
6683                 return phy <= PHY_B;
6684
6685         return false;
6686 }
6687
6688 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
6689 {
6690         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
6691                 return phy >= PHY_C && phy <= PHY_F;
6692
6693         return false;
6694 }
6695
6696 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
6697 {
6698         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
6699                 return PHY_A;
6700
6701         return (enum phy)port;
6702 }
6703
6704 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
6705 {
6706         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
6707                 return PORT_TC_NONE;
6708
6709         return port - PORT_C;
6710 }
6711
6712 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
6713 {
6714         switch (port) {
6715         case PORT_A:
6716                 return POWER_DOMAIN_PORT_DDI_A_LANES;
6717         case PORT_B:
6718                 return POWER_DOMAIN_PORT_DDI_B_LANES;
6719         case PORT_C:
6720                 return POWER_DOMAIN_PORT_DDI_C_LANES;
6721         case PORT_D:
6722                 return POWER_DOMAIN_PORT_DDI_D_LANES;
6723         case PORT_E:
6724                 return POWER_DOMAIN_PORT_DDI_E_LANES;
6725         case PORT_F:
6726                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6727         default:
6728                 MISSING_CASE(port);
6729                 return POWER_DOMAIN_PORT_OTHER;
6730         }
6731 }
6732
6733 enum intel_display_power_domain
6734 intel_aux_power_domain(struct intel_digital_port *dig_port)
6735 {
6736         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
6737         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
6738
6739         if (intel_phy_is_tc(dev_priv, phy) &&
6740             dig_port->tc_mode == TC_PORT_TBT_ALT) {
6741                 switch (dig_port->aux_ch) {
6742                 case AUX_CH_C:
6743                         return POWER_DOMAIN_AUX_TBT1;
6744                 case AUX_CH_D:
6745                         return POWER_DOMAIN_AUX_TBT2;
6746                 case AUX_CH_E:
6747                         return POWER_DOMAIN_AUX_TBT3;
6748                 case AUX_CH_F:
6749                         return POWER_DOMAIN_AUX_TBT4;
6750                 default:
6751                         MISSING_CASE(dig_port->aux_ch);
6752                         return POWER_DOMAIN_AUX_TBT1;
6753                 }
6754         }
6755
6756         switch (dig_port->aux_ch) {
6757         case AUX_CH_A:
6758                 return POWER_DOMAIN_AUX_A;
6759         case AUX_CH_B:
6760                 return POWER_DOMAIN_AUX_B;
6761         case AUX_CH_C:
6762                 return POWER_DOMAIN_AUX_C;
6763         case AUX_CH_D:
6764                 return POWER_DOMAIN_AUX_D;
6765         case AUX_CH_E:
6766                 return POWER_DOMAIN_AUX_E;
6767         case AUX_CH_F:
6768                 return POWER_DOMAIN_AUX_F;
6769         default:
6770                 MISSING_CASE(dig_port->aux_ch);
6771                 return POWER_DOMAIN_AUX_A;
6772         }
6773 }
6774
6775 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6776 {
6777         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6778         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6779         struct drm_encoder *encoder;
6780         enum pipe pipe = crtc->pipe;
6781         u64 mask;
6782         enum transcoder transcoder = crtc_state->cpu_transcoder;
6783
6784         if (!crtc_state->base.active)
6785                 return 0;
6786
6787         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6788         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6789         if (crtc_state->pch_pfit.enabled ||
6790             crtc_state->pch_pfit.force_thru)
6791                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6792
6793         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
6794                                   crtc_state->base.encoder_mask) {
6795                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6796
6797                 mask |= BIT_ULL(intel_encoder->power_domain);
6798         }
6799
6800         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6801                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6802
6803         if (crtc_state->shared_dpll)
6804                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
6805
6806         return mask;
6807 }
6808
6809 static u64
6810 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6811 {
6812         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6813         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6814         enum intel_display_power_domain domain;
6815         u64 domains, new_domains, old_domains;
6816
6817         old_domains = crtc->enabled_power_domains;
6818         crtc->enabled_power_domains = new_domains =
6819                 get_crtc_power_domains(crtc_state);
6820
6821         domains = new_domains & ~old_domains;
6822
6823         for_each_power_domain(domain, domains)
6824                 intel_display_power_get(dev_priv, domain);
6825
6826         return old_domains & ~new_domains;
6827 }
6828
6829 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6830                                       u64 domains)
6831 {
6832         enum intel_display_power_domain domain;
6833
6834         for_each_power_domain(domain, domains)
6835                 intel_display_power_put_unchecked(dev_priv, domain);
6836 }
6837
6838 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6839                                    struct intel_atomic_state *state)
6840 {
6841         struct drm_crtc *crtc = pipe_config->base.crtc;
6842         struct drm_device *dev = crtc->dev;
6843         struct drm_i915_private *dev_priv = to_i915(dev);
6844         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6845         int pipe = intel_crtc->pipe;
6846
6847         if (WARN_ON(intel_crtc->active))
6848                 return;
6849
6850         if (intel_crtc_has_dp_encoder(pipe_config))
6851                 intel_dp_set_m_n(pipe_config, M1_N1);
6852
6853         intel_set_pipe_timings(pipe_config);
6854         intel_set_pipe_src_size(pipe_config);
6855
6856         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6857                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6858                 I915_WRITE(CHV_CANVAS(pipe), 0);
6859         }
6860
6861         i9xx_set_pipeconf(pipe_config);
6862
6863         intel_crtc->active = true;
6864
6865         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6866
6867         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6868
6869         if (IS_CHERRYVIEW(dev_priv)) {
6870                 chv_prepare_pll(intel_crtc, pipe_config);
6871                 chv_enable_pll(intel_crtc, pipe_config);
6872         } else {
6873                 vlv_prepare_pll(intel_crtc, pipe_config);
6874                 vlv_enable_pll(intel_crtc, pipe_config);
6875         }
6876
6877         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6878
6879         i9xx_pfit_enable(pipe_config);
6880
6881         intel_color_load_luts(pipe_config);
6882         intel_color_commit(pipe_config);
6883         /* update DSPCNTR to configure gamma for pipe bottom color */
6884         intel_disable_primary_plane(pipe_config);
6885
6886         dev_priv->display.initial_watermarks(state, pipe_config);
6887         intel_enable_pipe(pipe_config);
6888
6889         assert_vblank_disabled(crtc);
6890         intel_crtc_vblank_on(pipe_config);
6891
6892         intel_encoders_enable(intel_crtc, pipe_config, state);
6893 }
6894
6895 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6896 {
6897         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6898         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6899
6900         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6901         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6902 }
6903
6904 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6905                              struct intel_atomic_state *state)
6906 {
6907         struct drm_crtc *crtc = pipe_config->base.crtc;
6908         struct drm_device *dev = crtc->dev;
6909         struct drm_i915_private *dev_priv = to_i915(dev);
6910         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6911         enum pipe pipe = intel_crtc->pipe;
6912
6913         if (WARN_ON(intel_crtc->active))
6914                 return;
6915
6916         i9xx_set_pll_dividers(pipe_config);
6917
6918         if (intel_crtc_has_dp_encoder(pipe_config))
6919                 intel_dp_set_m_n(pipe_config, M1_N1);
6920
6921         intel_set_pipe_timings(pipe_config);
6922         intel_set_pipe_src_size(pipe_config);
6923
6924         i9xx_set_pipeconf(pipe_config);
6925
6926         intel_crtc->active = true;
6927
6928         if (!IS_GEN(dev_priv, 2))
6929                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6930
6931         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6932
6933         i9xx_enable_pll(intel_crtc, pipe_config);
6934
6935         i9xx_pfit_enable(pipe_config);
6936
6937         intel_color_load_luts(pipe_config);
6938         intel_color_commit(pipe_config);
6939         /* update DSPCNTR to configure gamma for pipe bottom color */
6940         intel_disable_primary_plane(pipe_config);
6941
6942         if (dev_priv->display.initial_watermarks != NULL)
6943                 dev_priv->display.initial_watermarks(state,
6944                                                      pipe_config);
6945         else
6946                 intel_update_watermarks(intel_crtc);
6947         intel_enable_pipe(pipe_config);
6948
6949         assert_vblank_disabled(crtc);
6950         intel_crtc_vblank_on(pipe_config);
6951
6952         intel_encoders_enable(intel_crtc, pipe_config, state);
6953 }
6954
6955 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6956 {
6957         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6958         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6959
6960         if (!old_crtc_state->gmch_pfit.control)
6961                 return;
6962
6963         assert_pipe_disabled(dev_priv, crtc->pipe);
6964
6965         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6966                       I915_READ(PFIT_CONTROL));
6967         I915_WRITE(PFIT_CONTROL, 0);
6968 }
6969
6970 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6971                               struct intel_atomic_state *state)
6972 {
6973         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6974         struct drm_device *dev = crtc->dev;
6975         struct drm_i915_private *dev_priv = to_i915(dev);
6976         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6977         int pipe = intel_crtc->pipe;
6978
6979         /*
6980          * On gen2 planes are double buffered but the pipe isn't, so we must
6981          * wait for planes to fully turn off before disabling the pipe.
6982          */
6983         if (IS_GEN(dev_priv, 2))
6984                 intel_wait_for_vblank(dev_priv, pipe);
6985
6986         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6987
6988         drm_crtc_vblank_off(crtc);
6989         assert_vblank_disabled(crtc);
6990
6991         intel_disable_pipe(old_crtc_state);
6992
6993         i9xx_pfit_disable(old_crtc_state);
6994
6995         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6996
6997         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
6998                 if (IS_CHERRYVIEW(dev_priv))
6999                         chv_disable_pll(dev_priv, pipe);
7000                 else if (IS_VALLEYVIEW(dev_priv))
7001                         vlv_disable_pll(dev_priv, pipe);
7002                 else
7003                         i9xx_disable_pll(old_crtc_state);
7004         }
7005
7006         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
7007
7008         if (!IS_GEN(dev_priv, 2))
7009                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7010
7011         if (!dev_priv->display.initial_watermarks)
7012                 intel_update_watermarks(intel_crtc);
7013
7014         /* clock the pipe down to 640x480@60 to potentially save power */
7015         if (IS_I830(dev_priv))
7016                 i830_enable_pipe(dev_priv, pipe);
7017 }
7018
7019 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
7020                                         struct drm_modeset_acquire_ctx *ctx)
7021 {
7022         struct intel_encoder *encoder;
7023         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7024         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
7025         struct intel_bw_state *bw_state =
7026                 to_intel_bw_state(dev_priv->bw_obj.state);
7027         enum intel_display_power_domain domain;
7028         struct intel_plane *plane;
7029         u64 domains;
7030         struct drm_atomic_state *state;
7031         struct intel_crtc_state *crtc_state;
7032         int ret;
7033
7034         if (!intel_crtc->active)
7035                 return;
7036
7037         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
7038                 const struct intel_plane_state *plane_state =
7039                         to_intel_plane_state(plane->base.state);
7040
7041                 if (plane_state->base.visible)
7042                         intel_plane_disable_noatomic(intel_crtc, plane);
7043         }
7044
7045         state = drm_atomic_state_alloc(crtc->dev);
7046         if (!state) {
7047                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7048                               crtc->base.id, crtc->name);
7049                 return;
7050         }
7051
7052         state->acquire_ctx = ctx;
7053
7054         /* Everything's already locked, -EDEADLK can't happen. */
7055         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
7056         ret = drm_atomic_add_affected_connectors(state, crtc);
7057
7058         WARN_ON(IS_ERR(crtc_state) || ret);
7059
7060         dev_priv->display.crtc_disable(crtc_state, to_intel_atomic_state(state));
7061
7062         drm_atomic_state_put(state);
7063
7064         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7065                       crtc->base.id, crtc->name);
7066
7067         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
7068         crtc->state->active = false;
7069         intel_crtc->active = false;
7070         crtc->enabled = false;
7071         crtc->state->connector_mask = 0;
7072         crtc->state->encoder_mask = 0;
7073
7074         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
7075                 encoder->base.crtc = NULL;
7076
7077         intel_fbc_disable(intel_crtc);
7078         intel_update_watermarks(intel_crtc);
7079         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
7080
7081         domains = intel_crtc->enabled_power_domains;
7082         for_each_power_domain(domain, domains)
7083                 intel_display_power_put_unchecked(dev_priv, domain);
7084         intel_crtc->enabled_power_domains = 0;
7085
7086         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
7087         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
7088         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
7089
7090         bw_state->data_rate[intel_crtc->pipe] = 0;
7091         bw_state->num_active_planes[intel_crtc->pipe] = 0;
7092 }
7093
7094 /*
7095  * turn all crtc's off, but do not adjust state
7096  * This has to be paired with a call to intel_modeset_setup_hw_state.
7097  */
7098 int intel_display_suspend(struct drm_device *dev)
7099 {
7100         struct drm_i915_private *dev_priv = to_i915(dev);
7101         struct drm_atomic_state *state;
7102         int ret;
7103
7104         state = drm_atomic_helper_suspend(dev);
7105         ret = PTR_ERR_OR_ZERO(state);
7106         if (ret)
7107                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7108         else
7109                 dev_priv->modeset_restore_state = state;
7110         return ret;
7111 }
7112
7113 void intel_encoder_destroy(struct drm_encoder *encoder)
7114 {
7115         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7116
7117         drm_encoder_cleanup(encoder);
7118         kfree(intel_encoder);
7119 }
7120
7121 /* Cross check the actual hw state with our own modeset state tracking (and it's
7122  * internal consistency). */
7123 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7124                                          struct drm_connector_state *conn_state)
7125 {
7126         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7127
7128         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7129                       connector->base.base.id,
7130                       connector->base.name);
7131
7132         if (connector->get_hw_state(connector)) {
7133                 struct intel_encoder *encoder = connector->encoder;
7134
7135                 I915_STATE_WARN(!crtc_state,
7136                          "connector enabled without attached crtc\n");
7137
7138                 if (!crtc_state)
7139                         return;
7140
7141                 I915_STATE_WARN(!crtc_state->base.active,
7142                       "connector is active, but attached crtc isn't\n");
7143
7144                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7145                         return;
7146
7147                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7148                         "atomic encoder doesn't match attached encoder\n");
7149
7150                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7151                         "attached encoder crtc differs from connector crtc\n");
7152         } else {
7153                 I915_STATE_WARN(crtc_state && crtc_state->base.active,
7154                         "attached crtc is active, but connector isn't\n");
7155                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7156                         "best encoder set without crtc!\n");
7157         }
7158 }
7159
7160 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7161 {
7162         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
7163                 return crtc_state->fdi_lanes;
7164
7165         return 0;
7166 }
7167
7168 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7169                                      struct intel_crtc_state *pipe_config)
7170 {
7171         struct drm_i915_private *dev_priv = to_i915(dev);
7172         struct drm_atomic_state *state = pipe_config->base.state;
7173         struct intel_crtc *other_crtc;
7174         struct intel_crtc_state *other_crtc_state;
7175
7176         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7177                       pipe_name(pipe), pipe_config->fdi_lanes);
7178         if (pipe_config->fdi_lanes > 4) {
7179                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7180                               pipe_name(pipe), pipe_config->fdi_lanes);
7181                 return -EINVAL;
7182         }
7183
7184         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7185                 if (pipe_config->fdi_lanes > 2) {
7186                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7187                                       pipe_config->fdi_lanes);
7188                         return -EINVAL;
7189                 } else {
7190                         return 0;
7191                 }
7192         }
7193
7194         if (INTEL_INFO(dev_priv)->num_pipes == 2)
7195                 return 0;
7196
7197         /* Ivybridge 3 pipe is really complicated */
7198         switch (pipe) {
7199         case PIPE_A:
7200                 return 0;
7201         case PIPE_B:
7202                 if (pipe_config->fdi_lanes <= 2)
7203                         return 0;
7204
7205                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7206                 other_crtc_state =
7207                         intel_atomic_get_crtc_state(state, other_crtc);
7208                 if (IS_ERR(other_crtc_state))
7209                         return PTR_ERR(other_crtc_state);
7210
7211                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7212                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7213                                       pipe_name(pipe), pipe_config->fdi_lanes);
7214                         return -EINVAL;
7215                 }
7216                 return 0;
7217         case PIPE_C:
7218                 if (pipe_config->fdi_lanes > 2) {
7219                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7220                                       pipe_name(pipe), pipe_config->fdi_lanes);
7221                         return -EINVAL;
7222                 }
7223
7224                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7225                 other_crtc_state =
7226                         intel_atomic_get_crtc_state(state, other_crtc);
7227                 if (IS_ERR(other_crtc_state))
7228                         return PTR_ERR(other_crtc_state);
7229
7230                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7231                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7232                         return -EINVAL;
7233                 }
7234                 return 0;
7235         default:
7236                 BUG();
7237         }
7238 }
7239
7240 #define RETRY 1
7241 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7242                                        struct intel_crtc_state *pipe_config)
7243 {
7244         struct drm_device *dev = intel_crtc->base.dev;
7245         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7246         int lane, link_bw, fdi_dotclock, ret;
7247         bool needs_recompute = false;
7248
7249 retry:
7250         /* FDI is a binary signal running at ~2.7GHz, encoding
7251          * each output octet as 10 bits. The actual frequency
7252          * is stored as a divider into a 100MHz clock, and the
7253          * mode pixel clock is stored in units of 1KHz.
7254          * Hence the bw of each lane in terms of the mode signal
7255          * is:
7256          */
7257         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7258
7259         fdi_dotclock = adjusted_mode->crtc_clock;
7260
7261         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7262                                            pipe_config->pipe_bpp);
7263
7264         pipe_config->fdi_lanes = lane;
7265
7266         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7267                                link_bw, &pipe_config->fdi_m_n, false);
7268
7269         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7270         if (ret == -EDEADLK)
7271                 return ret;
7272
7273         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7274                 pipe_config->pipe_bpp -= 2*3;
7275                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7276                               pipe_config->pipe_bpp);
7277                 needs_recompute = true;
7278                 pipe_config->bw_constrained = true;
7279
7280                 goto retry;
7281         }
7282
7283         if (needs_recompute)
7284                 return RETRY;
7285
7286         return ret;
7287 }
7288
7289 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7290 {
7291         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7292         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7293
7294         /* IPS only exists on ULT machines and is tied to pipe A. */
7295         if (!hsw_crtc_supports_ips(crtc))
7296                 return false;
7297
7298         if (!i915_modparams.enable_ips)
7299                 return false;
7300
7301         if (crtc_state->pipe_bpp > 24)
7302                 return false;
7303
7304         /*
7305          * We compare against max which means we must take
7306          * the increased cdclk requirement into account when
7307          * calculating the new cdclk.
7308          *
7309          * Should measure whether using a lower cdclk w/o IPS
7310          */
7311         if (IS_BROADWELL(dev_priv) &&
7312             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7313                 return false;
7314
7315         return true;
7316 }
7317
7318 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7319 {
7320         struct drm_i915_private *dev_priv =
7321                 to_i915(crtc_state->base.crtc->dev);
7322         struct intel_atomic_state *intel_state =
7323                 to_intel_atomic_state(crtc_state->base.state);
7324
7325         if (!hsw_crtc_state_ips_capable(crtc_state))
7326                 return false;
7327
7328         /*
7329          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7330          * enabled and disabled dynamically based on package C states,
7331          * user space can't make reliable use of the CRCs, so let's just
7332          * completely disable it.
7333          */
7334         if (crtc_state->crc_enabled)
7335                 return false;
7336
7337         /* IPS should be fine as long as at least one plane is enabled. */
7338         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7339                 return false;
7340
7341         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7342         if (IS_BROADWELL(dev_priv) &&
7343             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7344                 return false;
7345
7346         return true;
7347 }
7348
7349 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7350 {
7351         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7352
7353         /* GDG double wide on either pipe, otherwise pipe A only */
7354         return INTEL_GEN(dev_priv) < 4 &&
7355                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7356 }
7357
7358 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7359 {
7360         u32 pixel_rate;
7361
7362         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
7363
7364         /*
7365          * We only use IF-ID interlacing. If we ever use
7366          * PF-ID we'll need to adjust the pixel_rate here.
7367          */
7368
7369         if (pipe_config->pch_pfit.enabled) {
7370                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7371                 u32 pfit_size = pipe_config->pch_pfit.size;
7372
7373                 pipe_w = pipe_config->pipe_src_w;
7374                 pipe_h = pipe_config->pipe_src_h;
7375
7376                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7377                 pfit_h = pfit_size & 0xFFFF;
7378                 if (pipe_w < pfit_w)
7379                         pipe_w = pfit_w;
7380                 if (pipe_h < pfit_h)
7381                         pipe_h = pfit_h;
7382
7383                 if (WARN_ON(!pfit_w || !pfit_h))
7384                         return pixel_rate;
7385
7386                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7387                                      pfit_w * pfit_h);
7388         }
7389
7390         return pixel_rate;
7391 }
7392
7393 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7394 {
7395         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
7396
7397         if (HAS_GMCH(dev_priv))
7398                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7399                 crtc_state->pixel_rate =
7400                         crtc_state->base.adjusted_mode.crtc_clock;
7401         else
7402                 crtc_state->pixel_rate =
7403                         ilk_pipe_pixel_rate(crtc_state);
7404 }
7405
7406 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7407                                      struct intel_crtc_state *pipe_config)
7408 {
7409         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7410         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7411         int clock_limit = dev_priv->max_dotclk_freq;
7412
7413         if (INTEL_GEN(dev_priv) < 4) {
7414                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7415
7416                 /*
7417                  * Enable double wide mode when the dot clock
7418                  * is > 90% of the (display) core speed.
7419                  */
7420                 if (intel_crtc_supports_double_wide(crtc) &&
7421                     adjusted_mode->crtc_clock > clock_limit) {
7422                         clock_limit = dev_priv->max_dotclk_freq;
7423                         pipe_config->double_wide = true;
7424                 }
7425         }
7426
7427         if (adjusted_mode->crtc_clock > clock_limit) {
7428                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7429                               adjusted_mode->crtc_clock, clock_limit,
7430                               yesno(pipe_config->double_wide));
7431                 return -EINVAL;
7432         }
7433
7434         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7435              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7436              pipe_config->base.ctm) {
7437                 /*
7438                  * There is only one pipe CSC unit per pipe, and we need that
7439                  * for output conversion from RGB->YCBCR. So if CTM is already
7440                  * applied we can't support YCBCR420 output.
7441                  */
7442                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7443                 return -EINVAL;
7444         }
7445
7446         /*
7447          * Pipe horizontal size must be even in:
7448          * - DVO ganged mode
7449          * - LVDS dual channel mode
7450          * - Double wide pipe
7451          */
7452         if (pipe_config->pipe_src_w & 1) {
7453                 if (pipe_config->double_wide) {
7454                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7455                         return -EINVAL;
7456                 }
7457
7458                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7459                     intel_is_dual_link_lvds(dev_priv)) {
7460                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7461                         return -EINVAL;
7462                 }
7463         }
7464
7465         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7466          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7467          */
7468         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7469                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7470                 return -EINVAL;
7471
7472         intel_crtc_compute_pixel_rate(pipe_config);
7473
7474         if (pipe_config->has_pch_encoder)
7475                 return ironlake_fdi_compute_config(crtc, pipe_config);
7476
7477         return 0;
7478 }
7479
7480 static void
7481 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7482 {
7483         while (*num > DATA_LINK_M_N_MASK ||
7484                *den > DATA_LINK_M_N_MASK) {
7485                 *num >>= 1;
7486                 *den >>= 1;
7487         }
7488 }
7489
7490 static void compute_m_n(unsigned int m, unsigned int n,
7491                         u32 *ret_m, u32 *ret_n,
7492                         bool constant_n)
7493 {
7494         /*
7495          * Several DP dongles in particular seem to be fussy about
7496          * too large link M/N values. Give N value as 0x8000 that
7497          * should be acceptable by specific devices. 0x8000 is the
7498          * specified fixed N value for asynchronous clock mode,
7499          * which the devices expect also in synchronous clock mode.
7500          */
7501         if (constant_n)
7502                 *ret_n = 0x8000;
7503         else
7504                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7505
7506         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7507         intel_reduce_m_n_ratio(ret_m, ret_n);
7508 }
7509
7510 void
7511 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7512                        int pixel_clock, int link_clock,
7513                        struct intel_link_m_n *m_n,
7514                        bool constant_n)
7515 {
7516         m_n->tu = 64;
7517
7518         compute_m_n(bits_per_pixel * pixel_clock,
7519                     link_clock * nlanes * 8,
7520                     &m_n->gmch_m, &m_n->gmch_n,
7521                     constant_n);
7522
7523         compute_m_n(pixel_clock, link_clock,
7524                     &m_n->link_m, &m_n->link_n,
7525                     constant_n);
7526 }
7527
7528 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7529 {
7530         if (i915_modparams.panel_use_ssc >= 0)
7531                 return i915_modparams.panel_use_ssc != 0;
7532         return dev_priv->vbt.lvds_use_ssc
7533                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7534 }
7535
7536 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7537 {
7538         return (1 << dpll->n) << 16 | dpll->m2;
7539 }
7540
7541 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7542 {
7543         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7544 }
7545
7546 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7547                                      struct intel_crtc_state *crtc_state,
7548                                      struct dpll *reduced_clock)
7549 {
7550         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7551         u32 fp, fp2 = 0;
7552
7553         if (IS_PINEVIEW(dev_priv)) {
7554                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7555                 if (reduced_clock)
7556                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7557         } else {
7558                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7559                 if (reduced_clock)
7560                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7561         }
7562
7563         crtc_state->dpll_hw_state.fp0 = fp;
7564
7565         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7566             reduced_clock) {
7567                 crtc_state->dpll_hw_state.fp1 = fp2;
7568         } else {
7569                 crtc_state->dpll_hw_state.fp1 = fp;
7570         }
7571 }
7572
7573 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7574                 pipe)
7575 {
7576         u32 reg_val;
7577
7578         /*
7579          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7580          * and set it to a reasonable value instead.
7581          */
7582         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7583         reg_val &= 0xffffff00;
7584         reg_val |= 0x00000030;
7585         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7586
7587         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7588         reg_val &= 0x00ffffff;
7589         reg_val |= 0x8c000000;
7590         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7591
7592         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7593         reg_val &= 0xffffff00;
7594         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7595
7596         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7597         reg_val &= 0x00ffffff;
7598         reg_val |= 0xb0000000;
7599         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7600 }
7601
7602 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7603                                          const struct intel_link_m_n *m_n)
7604 {
7605         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7606         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7607         enum pipe pipe = crtc->pipe;
7608
7609         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7610         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7611         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7612         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7613 }
7614
7615 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7616                                  enum transcoder transcoder)
7617 {
7618         if (IS_HASWELL(dev_priv))
7619                 return transcoder == TRANSCODER_EDP;
7620
7621         /*
7622          * Strictly speaking some registers are available before
7623          * gen7, but we only support DRRS on gen7+
7624          */
7625         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7626 }
7627
7628 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7629                                          const struct intel_link_m_n *m_n,
7630                                          const struct intel_link_m_n *m2_n2)
7631 {
7632         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7633         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7634         enum pipe pipe = crtc->pipe;
7635         enum transcoder transcoder = crtc_state->cpu_transcoder;
7636
7637         if (INTEL_GEN(dev_priv) >= 5) {
7638                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7639                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7640                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7641                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7642                 /*
7643                  *  M2_N2 registers are set only if DRRS is supported
7644                  * (to make sure the registers are not unnecessarily accessed).
7645                  */
7646                 if (m2_n2 && crtc_state->has_drrs &&
7647                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7648                         I915_WRITE(PIPE_DATA_M2(transcoder),
7649                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7650                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7651                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7652                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7653                 }
7654         } else {
7655                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7656                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7657                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7658                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7659         }
7660 }
7661
7662 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
7663 {
7664         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7665
7666         if (m_n == M1_N1) {
7667                 dp_m_n = &crtc_state->dp_m_n;
7668                 dp_m2_n2 = &crtc_state->dp_m2_n2;
7669         } else if (m_n == M2_N2) {
7670
7671                 /*
7672                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7673                  * needs to be programmed into M1_N1.
7674                  */
7675                 dp_m_n = &crtc_state->dp_m2_n2;
7676         } else {
7677                 DRM_ERROR("Unsupported divider value\n");
7678                 return;
7679         }
7680
7681         if (crtc_state->has_pch_encoder)
7682                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
7683         else
7684                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
7685 }
7686
7687 static void vlv_compute_dpll(struct intel_crtc *crtc,
7688                              struct intel_crtc_state *pipe_config)
7689 {
7690         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7691                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7692         if (crtc->pipe != PIPE_A)
7693                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7694
7695         /* DPLL not used with DSI, but still need the rest set up */
7696         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7697                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7698                         DPLL_EXT_BUFFER_ENABLE_VLV;
7699
7700         pipe_config->dpll_hw_state.dpll_md =
7701                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7702 }
7703
7704 static void chv_compute_dpll(struct intel_crtc *crtc,
7705                              struct intel_crtc_state *pipe_config)
7706 {
7707         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7708                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7709         if (crtc->pipe != PIPE_A)
7710                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7711
7712         /* DPLL not used with DSI, but still need the rest set up */
7713         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7714                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7715
7716         pipe_config->dpll_hw_state.dpll_md =
7717                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7718 }
7719
7720 static void vlv_prepare_pll(struct intel_crtc *crtc,
7721                             const struct intel_crtc_state *pipe_config)
7722 {
7723         struct drm_device *dev = crtc->base.dev;
7724         struct drm_i915_private *dev_priv = to_i915(dev);
7725         enum pipe pipe = crtc->pipe;
7726         u32 mdiv;
7727         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7728         u32 coreclk, reg_val;
7729
7730         /* Enable Refclk */
7731         I915_WRITE(DPLL(pipe),
7732                    pipe_config->dpll_hw_state.dpll &
7733                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7734
7735         /* No need to actually set up the DPLL with DSI */
7736         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7737                 return;
7738
7739         vlv_dpio_get(dev_priv);
7740
7741         bestn = pipe_config->dpll.n;
7742         bestm1 = pipe_config->dpll.m1;
7743         bestm2 = pipe_config->dpll.m2;
7744         bestp1 = pipe_config->dpll.p1;
7745         bestp2 = pipe_config->dpll.p2;
7746
7747         /* See eDP HDMI DPIO driver vbios notes doc */
7748
7749         /* PLL B needs special handling */
7750         if (pipe == PIPE_B)
7751                 vlv_pllb_recal_opamp(dev_priv, pipe);
7752
7753         /* Set up Tx target for periodic Rcomp update */
7754         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7755
7756         /* Disable target IRef on PLL */
7757         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7758         reg_val &= 0x00ffffff;
7759         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7760
7761         /* Disable fast lock */
7762         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7763
7764         /* Set idtafcrecal before PLL is enabled */
7765         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7766         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7767         mdiv |= ((bestn << DPIO_N_SHIFT));
7768         mdiv |= (1 << DPIO_K_SHIFT);
7769
7770         /*
7771          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7772          * but we don't support that).
7773          * Note: don't use the DAC post divider as it seems unstable.
7774          */
7775         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7776         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7777
7778         mdiv |= DPIO_ENABLE_CALIBRATION;
7779         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7780
7781         /* Set HBR and RBR LPF coefficients */
7782         if (pipe_config->port_clock == 162000 ||
7783             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7784             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7785                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7786                                  0x009f0003);
7787         else
7788                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7789                                  0x00d0000f);
7790
7791         if (intel_crtc_has_dp_encoder(pipe_config)) {
7792                 /* Use SSC source */
7793                 if (pipe == PIPE_A)
7794                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7795                                          0x0df40000);
7796                 else
7797                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7798                                          0x0df70000);
7799         } else { /* HDMI or VGA */
7800                 /* Use bend source */
7801                 if (pipe == PIPE_A)
7802                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7803                                          0x0df70000);
7804                 else
7805                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7806                                          0x0df40000);
7807         }
7808
7809         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7810         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7811         if (intel_crtc_has_dp_encoder(pipe_config))
7812                 coreclk |= 0x01000000;
7813         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7814
7815         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7816
7817         vlv_dpio_put(dev_priv);
7818 }
7819
7820 static void chv_prepare_pll(struct intel_crtc *crtc,
7821                             const struct intel_crtc_state *pipe_config)
7822 {
7823         struct drm_device *dev = crtc->base.dev;
7824         struct drm_i915_private *dev_priv = to_i915(dev);
7825         enum pipe pipe = crtc->pipe;
7826         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7827         u32 loopfilter, tribuf_calcntr;
7828         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7829         u32 dpio_val;
7830         int vco;
7831
7832         /* Enable Refclk and SSC */
7833         I915_WRITE(DPLL(pipe),
7834                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7835
7836         /* No need to actually set up the DPLL with DSI */
7837         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7838                 return;
7839
7840         bestn = pipe_config->dpll.n;
7841         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7842         bestm1 = pipe_config->dpll.m1;
7843         bestm2 = pipe_config->dpll.m2 >> 22;
7844         bestp1 = pipe_config->dpll.p1;
7845         bestp2 = pipe_config->dpll.p2;
7846         vco = pipe_config->dpll.vco;
7847         dpio_val = 0;
7848         loopfilter = 0;
7849
7850         vlv_dpio_get(dev_priv);
7851
7852         /* p1 and p2 divider */
7853         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7854                         5 << DPIO_CHV_S1_DIV_SHIFT |
7855                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7856                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7857                         1 << DPIO_CHV_K_DIV_SHIFT);
7858
7859         /* Feedback post-divider - m2 */
7860         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7861
7862         /* Feedback refclk divider - n and m1 */
7863         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7864                         DPIO_CHV_M1_DIV_BY_2 |
7865                         1 << DPIO_CHV_N_DIV_SHIFT);
7866
7867         /* M2 fraction division */
7868         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7869
7870         /* M2 fraction division enable */
7871         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7872         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7873         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7874         if (bestm2_frac)
7875                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7876         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7877
7878         /* Program digital lock detect threshold */
7879         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7880         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7881                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7882         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7883         if (!bestm2_frac)
7884                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7885         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7886
7887         /* Loop filter */
7888         if (vco == 5400000) {
7889                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7890                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7891                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7892                 tribuf_calcntr = 0x9;
7893         } else if (vco <= 6200000) {
7894                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7895                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7896                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7897                 tribuf_calcntr = 0x9;
7898         } else if (vco <= 6480000) {
7899                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7900                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7901                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7902                 tribuf_calcntr = 0x8;
7903         } else {
7904                 /* Not supported. Apply the same limits as in the max case */
7905                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7906                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7907                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7908                 tribuf_calcntr = 0;
7909         }
7910         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7911
7912         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7913         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7914         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7915         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7916
7917         /* AFC Recal */
7918         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7919                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7920                         DPIO_AFC_RECAL);
7921
7922         vlv_dpio_put(dev_priv);
7923 }
7924
7925 /**
7926  * vlv_force_pll_on - forcibly enable just the PLL
7927  * @dev_priv: i915 private structure
7928  * @pipe: pipe PLL to enable
7929  * @dpll: PLL configuration
7930  *
7931  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7932  * in cases where we need the PLL enabled even when @pipe is not going to
7933  * be enabled.
7934  */
7935 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7936                      const struct dpll *dpll)
7937 {
7938         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7939         struct intel_crtc_state *pipe_config;
7940
7941         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7942         if (!pipe_config)
7943                 return -ENOMEM;
7944
7945         pipe_config->base.crtc = &crtc->base;
7946         pipe_config->pixel_multiplier = 1;
7947         pipe_config->dpll = *dpll;
7948
7949         if (IS_CHERRYVIEW(dev_priv)) {
7950                 chv_compute_dpll(crtc, pipe_config);
7951                 chv_prepare_pll(crtc, pipe_config);
7952                 chv_enable_pll(crtc, pipe_config);
7953         } else {
7954                 vlv_compute_dpll(crtc, pipe_config);
7955                 vlv_prepare_pll(crtc, pipe_config);
7956                 vlv_enable_pll(crtc, pipe_config);
7957         }
7958
7959         kfree(pipe_config);
7960
7961         return 0;
7962 }
7963
7964 /**
7965  * vlv_force_pll_off - forcibly disable just the PLL
7966  * @dev_priv: i915 private structure
7967  * @pipe: pipe PLL to disable
7968  *
7969  * Disable the PLL for @pipe. To be used in cases where we need
7970  * the PLL enabled even when @pipe is not going to be enabled.
7971  */
7972 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7973 {
7974         if (IS_CHERRYVIEW(dev_priv))
7975                 chv_disable_pll(dev_priv, pipe);
7976         else
7977                 vlv_disable_pll(dev_priv, pipe);
7978 }
7979
7980 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7981                               struct intel_crtc_state *crtc_state,
7982                               struct dpll *reduced_clock)
7983 {
7984         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7985         u32 dpll;
7986         struct dpll *clock = &crtc_state->dpll;
7987
7988         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7989
7990         dpll = DPLL_VGA_MODE_DIS;
7991
7992         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7993                 dpll |= DPLLB_MODE_LVDS;
7994         else
7995                 dpll |= DPLLB_MODE_DAC_SERIAL;
7996
7997         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7998             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7999                 dpll |= (crtc_state->pixel_multiplier - 1)
8000                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8001         }
8002
8003         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8004             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8005                 dpll |= DPLL_SDVO_HIGH_SPEED;
8006
8007         if (intel_crtc_has_dp_encoder(crtc_state))
8008                 dpll |= DPLL_SDVO_HIGH_SPEED;
8009
8010         /* compute bitmask from p1 value */
8011         if (IS_PINEVIEW(dev_priv))
8012                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8013         else {
8014                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8015                 if (IS_G4X(dev_priv) && reduced_clock)
8016                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8017         }
8018         switch (clock->p2) {
8019         case 5:
8020                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8021                 break;
8022         case 7:
8023                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8024                 break;
8025         case 10:
8026                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8027                 break;
8028         case 14:
8029                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8030                 break;
8031         }
8032         if (INTEL_GEN(dev_priv) >= 4)
8033                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8034
8035         if (crtc_state->sdvo_tv_clock)
8036                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8037         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8038                  intel_panel_use_ssc(dev_priv))
8039                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8040         else
8041                 dpll |= PLL_REF_INPUT_DREFCLK;
8042
8043         dpll |= DPLL_VCO_ENABLE;
8044         crtc_state->dpll_hw_state.dpll = dpll;
8045
8046         if (INTEL_GEN(dev_priv) >= 4) {
8047                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8048                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8049                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8050         }
8051 }
8052
8053 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8054                               struct intel_crtc_state *crtc_state,
8055                               struct dpll *reduced_clock)
8056 {
8057         struct drm_device *dev = crtc->base.dev;
8058         struct drm_i915_private *dev_priv = to_i915(dev);
8059         u32 dpll;
8060         struct dpll *clock = &crtc_state->dpll;
8061
8062         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8063
8064         dpll = DPLL_VGA_MODE_DIS;
8065
8066         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8067                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8068         } else {
8069                 if (clock->p1 == 2)
8070                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8071                 else
8072                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8073                 if (clock->p2 == 4)
8074                         dpll |= PLL_P2_DIVIDE_BY_4;
8075         }
8076
8077         /*
8078          * Bspec:
8079          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8080          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8081          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8082          *  Enable) must be set to “1” in both the DPLL A Control Register
8083          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8084          *
8085          * For simplicity We simply keep both bits always enabled in
8086          * both DPLLS. The spec says we should disable the DVO 2X clock
8087          * when not needed, but this seems to work fine in practice.
8088          */
8089         if (IS_I830(dev_priv) ||
8090             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8091                 dpll |= DPLL_DVO_2X_MODE;
8092
8093         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8094             intel_panel_use_ssc(dev_priv))
8095                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8096         else
8097                 dpll |= PLL_REF_INPUT_DREFCLK;
8098
8099         dpll |= DPLL_VCO_ENABLE;
8100         crtc_state->dpll_hw_state.dpll = dpll;
8101 }
8102
8103 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8104 {
8105         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8106         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8107         enum pipe pipe = crtc->pipe;
8108         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8109         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
8110         u32 crtc_vtotal, crtc_vblank_end;
8111         int vsyncshift = 0;
8112
8113         /* We need to be careful not to changed the adjusted mode, for otherwise
8114          * the hw state checker will get angry at the mismatch. */
8115         crtc_vtotal = adjusted_mode->crtc_vtotal;
8116         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8117
8118         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8119                 /* the chip adds 2 halflines automatically */
8120                 crtc_vtotal -= 1;
8121                 crtc_vblank_end -= 1;
8122
8123                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8124                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8125                 else
8126                         vsyncshift = adjusted_mode->crtc_hsync_start -
8127                                 adjusted_mode->crtc_htotal / 2;
8128                 if (vsyncshift < 0)
8129                         vsyncshift += adjusted_mode->crtc_htotal;
8130         }
8131
8132         if (INTEL_GEN(dev_priv) > 3)
8133                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8134
8135         I915_WRITE(HTOTAL(cpu_transcoder),
8136                    (adjusted_mode->crtc_hdisplay - 1) |
8137                    ((adjusted_mode->crtc_htotal - 1) << 16));
8138         I915_WRITE(HBLANK(cpu_transcoder),
8139                    (adjusted_mode->crtc_hblank_start - 1) |
8140                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8141         I915_WRITE(HSYNC(cpu_transcoder),
8142                    (adjusted_mode->crtc_hsync_start - 1) |
8143                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8144
8145         I915_WRITE(VTOTAL(cpu_transcoder),
8146                    (adjusted_mode->crtc_vdisplay - 1) |
8147                    ((crtc_vtotal - 1) << 16));
8148         I915_WRITE(VBLANK(cpu_transcoder),
8149                    (adjusted_mode->crtc_vblank_start - 1) |
8150                    ((crtc_vblank_end - 1) << 16));
8151         I915_WRITE(VSYNC(cpu_transcoder),
8152                    (adjusted_mode->crtc_vsync_start - 1) |
8153                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8154
8155         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8156          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8157          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8158          * bits. */
8159         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8160             (pipe == PIPE_B || pipe == PIPE_C))
8161                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8162
8163 }
8164
8165 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8166 {
8167         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8168         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8169         enum pipe pipe = crtc->pipe;
8170
8171         /* pipesrc controls the size that is scaled from, which should
8172          * always be the user's requested size.
8173          */
8174         I915_WRITE(PIPESRC(pipe),
8175                    ((crtc_state->pipe_src_w - 1) << 16) |
8176                    (crtc_state->pipe_src_h - 1));
8177 }
8178
8179 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8180                                    struct intel_crtc_state *pipe_config)
8181 {
8182         struct drm_device *dev = crtc->base.dev;
8183         struct drm_i915_private *dev_priv = to_i915(dev);
8184         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8185         u32 tmp;
8186
8187         tmp = I915_READ(HTOTAL(cpu_transcoder));
8188         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8189         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8190
8191         if (!transcoder_is_dsi(cpu_transcoder)) {
8192                 tmp = I915_READ(HBLANK(cpu_transcoder));
8193                 pipe_config->base.adjusted_mode.crtc_hblank_start =
8194                                                         (tmp & 0xffff) + 1;
8195                 pipe_config->base.adjusted_mode.crtc_hblank_end =
8196                                                 ((tmp >> 16) & 0xffff) + 1;
8197         }
8198         tmp = I915_READ(HSYNC(cpu_transcoder));
8199         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8200         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8201
8202         tmp = I915_READ(VTOTAL(cpu_transcoder));
8203         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8204         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8205
8206         if (!transcoder_is_dsi(cpu_transcoder)) {
8207                 tmp = I915_READ(VBLANK(cpu_transcoder));
8208                 pipe_config->base.adjusted_mode.crtc_vblank_start =
8209                                                         (tmp & 0xffff) + 1;
8210                 pipe_config->base.adjusted_mode.crtc_vblank_end =
8211                                                 ((tmp >> 16) & 0xffff) + 1;
8212         }
8213         tmp = I915_READ(VSYNC(cpu_transcoder));
8214         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8215         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8216
8217         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
8218                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8219                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
8220                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
8221         }
8222 }
8223
8224 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8225                                     struct intel_crtc_state *pipe_config)
8226 {
8227         struct drm_device *dev = crtc->base.dev;
8228         struct drm_i915_private *dev_priv = to_i915(dev);
8229         u32 tmp;
8230
8231         tmp = I915_READ(PIPESRC(crtc->pipe));
8232         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8233         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8234
8235         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
8236         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
8237 }
8238
8239 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8240                                  struct intel_crtc_state *pipe_config)
8241 {
8242         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
8243         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
8244         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
8245         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
8246
8247         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
8248         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
8249         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
8250         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
8251
8252         mode->flags = pipe_config->base.adjusted_mode.flags;
8253         mode->type = DRM_MODE_TYPE_DRIVER;
8254
8255         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
8256
8257         mode->hsync = drm_mode_hsync(mode);
8258         mode->vrefresh = drm_mode_vrefresh(mode);
8259         drm_mode_set_name(mode);
8260 }
8261
8262 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8263 {
8264         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8265         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8266         u32 pipeconf;
8267
8268         pipeconf = 0;
8269
8270         /* we keep both pipes enabled on 830 */
8271         if (IS_I830(dev_priv))
8272                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8273
8274         if (crtc_state->double_wide)
8275                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8276
8277         /* only g4x and later have fancy bpc/dither controls */
8278         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8279             IS_CHERRYVIEW(dev_priv)) {
8280                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8281                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8282                         pipeconf |= PIPECONF_DITHER_EN |
8283                                     PIPECONF_DITHER_TYPE_SP;
8284
8285                 switch (crtc_state->pipe_bpp) {
8286                 case 18:
8287                         pipeconf |= PIPECONF_6BPC;
8288                         break;
8289                 case 24:
8290                         pipeconf |= PIPECONF_8BPC;
8291                         break;
8292                 case 30:
8293                         pipeconf |= PIPECONF_10BPC;
8294                         break;
8295                 default:
8296                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8297                         BUG();
8298                 }
8299         }
8300
8301         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8302                 if (INTEL_GEN(dev_priv) < 4 ||
8303                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8304                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8305                 else
8306                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8307         } else {
8308                 pipeconf |= PIPECONF_PROGRESSIVE;
8309         }
8310
8311         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8312              crtc_state->limited_color_range)
8313                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8314
8315         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8316
8317         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8318         POSTING_READ(PIPECONF(crtc->pipe));
8319 }
8320
8321 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8322                                    struct intel_crtc_state *crtc_state)
8323 {
8324         struct drm_device *dev = crtc->base.dev;
8325         struct drm_i915_private *dev_priv = to_i915(dev);
8326         const struct intel_limit *limit;
8327         int refclk = 48000;
8328
8329         memset(&crtc_state->dpll_hw_state, 0,
8330                sizeof(crtc_state->dpll_hw_state));
8331
8332         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8333                 if (intel_panel_use_ssc(dev_priv)) {
8334                         refclk = dev_priv->vbt.lvds_ssc_freq;
8335                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8336                 }
8337
8338                 limit = &intel_limits_i8xx_lvds;
8339         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8340                 limit = &intel_limits_i8xx_dvo;
8341         } else {
8342                 limit = &intel_limits_i8xx_dac;
8343         }
8344
8345         if (!crtc_state->clock_set &&
8346             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8347                                  refclk, NULL, &crtc_state->dpll)) {
8348                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8349                 return -EINVAL;
8350         }
8351
8352         i8xx_compute_dpll(crtc, crtc_state, NULL);
8353
8354         return 0;
8355 }
8356
8357 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8358                                   struct intel_crtc_state *crtc_state)
8359 {
8360         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8361         const struct intel_limit *limit;
8362         int refclk = 96000;
8363
8364         memset(&crtc_state->dpll_hw_state, 0,
8365                sizeof(crtc_state->dpll_hw_state));
8366
8367         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8368                 if (intel_panel_use_ssc(dev_priv)) {
8369                         refclk = dev_priv->vbt.lvds_ssc_freq;
8370                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8371                 }
8372
8373                 if (intel_is_dual_link_lvds(dev_priv))
8374                         limit = &intel_limits_g4x_dual_channel_lvds;
8375                 else
8376                         limit = &intel_limits_g4x_single_channel_lvds;
8377         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8378                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8379                 limit = &intel_limits_g4x_hdmi;
8380         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8381                 limit = &intel_limits_g4x_sdvo;
8382         } else {
8383                 /* The option is for other outputs */
8384                 limit = &intel_limits_i9xx_sdvo;
8385         }
8386
8387         if (!crtc_state->clock_set &&
8388             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8389                                 refclk, NULL, &crtc_state->dpll)) {
8390                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8391                 return -EINVAL;
8392         }
8393
8394         i9xx_compute_dpll(crtc, crtc_state, NULL);
8395
8396         return 0;
8397 }
8398
8399 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8400                                   struct intel_crtc_state *crtc_state)
8401 {
8402         struct drm_device *dev = crtc->base.dev;
8403         struct drm_i915_private *dev_priv = to_i915(dev);
8404         const struct intel_limit *limit;
8405         int refclk = 96000;
8406
8407         memset(&crtc_state->dpll_hw_state, 0,
8408                sizeof(crtc_state->dpll_hw_state));
8409
8410         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8411                 if (intel_panel_use_ssc(dev_priv)) {
8412                         refclk = dev_priv->vbt.lvds_ssc_freq;
8413                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8414                 }
8415
8416                 limit = &intel_limits_pineview_lvds;
8417         } else {
8418                 limit = &intel_limits_pineview_sdvo;
8419         }
8420
8421         if (!crtc_state->clock_set &&
8422             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8423                                 refclk, NULL, &crtc_state->dpll)) {
8424                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8425                 return -EINVAL;
8426         }
8427
8428         i9xx_compute_dpll(crtc, crtc_state, NULL);
8429
8430         return 0;
8431 }
8432
8433 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8434                                    struct intel_crtc_state *crtc_state)
8435 {
8436         struct drm_device *dev = crtc->base.dev;
8437         struct drm_i915_private *dev_priv = to_i915(dev);
8438         const struct intel_limit *limit;
8439         int refclk = 96000;
8440
8441         memset(&crtc_state->dpll_hw_state, 0,
8442                sizeof(crtc_state->dpll_hw_state));
8443
8444         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8445                 if (intel_panel_use_ssc(dev_priv)) {
8446                         refclk = dev_priv->vbt.lvds_ssc_freq;
8447                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8448                 }
8449
8450                 limit = &intel_limits_i9xx_lvds;
8451         } else {
8452                 limit = &intel_limits_i9xx_sdvo;
8453         }
8454
8455         if (!crtc_state->clock_set &&
8456             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8457                                  refclk, NULL, &crtc_state->dpll)) {
8458                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8459                 return -EINVAL;
8460         }
8461
8462         i9xx_compute_dpll(crtc, crtc_state, NULL);
8463
8464         return 0;
8465 }
8466
8467 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8468                                   struct intel_crtc_state *crtc_state)
8469 {
8470         int refclk = 100000;
8471         const struct intel_limit *limit = &intel_limits_chv;
8472
8473         memset(&crtc_state->dpll_hw_state, 0,
8474                sizeof(crtc_state->dpll_hw_state));
8475
8476         if (!crtc_state->clock_set &&
8477             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8478                                 refclk, NULL, &crtc_state->dpll)) {
8479                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8480                 return -EINVAL;
8481         }
8482
8483         chv_compute_dpll(crtc, crtc_state);
8484
8485         return 0;
8486 }
8487
8488 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8489                                   struct intel_crtc_state *crtc_state)
8490 {
8491         int refclk = 100000;
8492         const struct intel_limit *limit = &intel_limits_vlv;
8493
8494         memset(&crtc_state->dpll_hw_state, 0,
8495                sizeof(crtc_state->dpll_hw_state));
8496
8497         if (!crtc_state->clock_set &&
8498             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8499                                 refclk, NULL, &crtc_state->dpll)) {
8500                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8501                 return -EINVAL;
8502         }
8503
8504         vlv_compute_dpll(crtc, crtc_state);
8505
8506         return 0;
8507 }
8508
8509 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8510 {
8511         if (IS_I830(dev_priv))
8512                 return false;
8513
8514         return INTEL_GEN(dev_priv) >= 4 ||
8515                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8516 }
8517
8518 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8519                                  struct intel_crtc_state *pipe_config)
8520 {
8521         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8522         u32 tmp;
8523
8524         if (!i9xx_has_pfit(dev_priv))
8525                 return;
8526
8527         tmp = I915_READ(PFIT_CONTROL);
8528         if (!(tmp & PFIT_ENABLE))
8529                 return;
8530
8531         /* Check whether the pfit is attached to our pipe. */
8532         if (INTEL_GEN(dev_priv) < 4) {
8533                 if (crtc->pipe != PIPE_B)
8534                         return;
8535         } else {
8536                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8537                         return;
8538         }
8539
8540         pipe_config->gmch_pfit.control = tmp;
8541         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8542 }
8543
8544 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8545                                struct intel_crtc_state *pipe_config)
8546 {
8547         struct drm_device *dev = crtc->base.dev;
8548         struct drm_i915_private *dev_priv = to_i915(dev);
8549         int pipe = pipe_config->cpu_transcoder;
8550         struct dpll clock;
8551         u32 mdiv;
8552         int refclk = 100000;
8553
8554         /* In case of DSI, DPLL will not be used */
8555         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8556                 return;
8557
8558         vlv_dpio_get(dev_priv);
8559         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8560         vlv_dpio_put(dev_priv);
8561
8562         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8563         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8564         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8565         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8566         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8567
8568         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8569 }
8570
8571 static void
8572 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8573                               struct intel_initial_plane_config *plane_config)
8574 {
8575         struct drm_device *dev = crtc->base.dev;
8576         struct drm_i915_private *dev_priv = to_i915(dev);
8577         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8578         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8579         enum pipe pipe;
8580         u32 val, base, offset;
8581         int fourcc, pixel_format;
8582         unsigned int aligned_height;
8583         struct drm_framebuffer *fb;
8584         struct intel_framebuffer *intel_fb;
8585
8586         if (!plane->get_hw_state(plane, &pipe))
8587                 return;
8588
8589         WARN_ON(pipe != crtc->pipe);
8590
8591         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8592         if (!intel_fb) {
8593                 DRM_DEBUG_KMS("failed to alloc fb\n");
8594                 return;
8595         }
8596
8597         fb = &intel_fb->base;
8598
8599         fb->dev = dev;
8600
8601         val = I915_READ(DSPCNTR(i9xx_plane));
8602
8603         if (INTEL_GEN(dev_priv) >= 4) {
8604                 if (val & DISPPLANE_TILED) {
8605                         plane_config->tiling = I915_TILING_X;
8606                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8607                 }
8608
8609                 if (val & DISPPLANE_ROTATE_180)
8610                         plane_config->rotation = DRM_MODE_ROTATE_180;
8611         }
8612
8613         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8614             val & DISPPLANE_MIRROR)
8615                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8616
8617         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8618         fourcc = i9xx_format_to_fourcc(pixel_format);
8619         fb->format = drm_format_info(fourcc);
8620
8621         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8622                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8623                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8624         } else if (INTEL_GEN(dev_priv) >= 4) {
8625                 if (plane_config->tiling)
8626                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8627                 else
8628                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8629                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8630         } else {
8631                 base = I915_READ(DSPADDR(i9xx_plane));
8632         }
8633         plane_config->base = base;
8634
8635         val = I915_READ(PIPESRC(pipe));
8636         fb->width = ((val >> 16) & 0xfff) + 1;
8637         fb->height = ((val >> 0) & 0xfff) + 1;
8638
8639         val = I915_READ(DSPSTRIDE(i9xx_plane));
8640         fb->pitches[0] = val & 0xffffffc0;
8641
8642         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8643
8644         plane_config->size = fb->pitches[0] * aligned_height;
8645
8646         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8647                       crtc->base.name, plane->base.name, fb->width, fb->height,
8648                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8649                       plane_config->size);
8650
8651         plane_config->fb = intel_fb;
8652 }
8653
8654 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8655                                struct intel_crtc_state *pipe_config)
8656 {
8657         struct drm_device *dev = crtc->base.dev;
8658         struct drm_i915_private *dev_priv = to_i915(dev);
8659         int pipe = pipe_config->cpu_transcoder;
8660         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8661         struct dpll clock;
8662         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8663         int refclk = 100000;
8664
8665         /* In case of DSI, DPLL will not be used */
8666         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8667                 return;
8668
8669         vlv_dpio_get(dev_priv);
8670         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8671         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8672         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8673         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8674         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8675         vlv_dpio_put(dev_priv);
8676
8677         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8678         clock.m2 = (pll_dw0 & 0xff) << 22;
8679         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8680                 clock.m2 |= pll_dw2 & 0x3fffff;
8681         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8682         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8683         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8684
8685         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8686 }
8687
8688 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
8689                                         struct intel_crtc_state *pipe_config)
8690 {
8691         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8692         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
8693
8694         pipe_config->lspcon_downsampling = false;
8695
8696         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8697                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
8698
8699                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
8700                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
8701                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
8702
8703                         if (ycbcr420_enabled) {
8704                                 /* We support 4:2:0 in full blend mode only */
8705                                 if (!blend)
8706                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8707                                 else if (!(IS_GEMINILAKE(dev_priv) ||
8708                                            INTEL_GEN(dev_priv) >= 10))
8709                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8710                                 else
8711                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
8712                         } else {
8713                                 /*
8714                                  * Currently there is no interface defined to
8715                                  * check user preference between RGB/YCBCR444
8716                                  * or YCBCR420. So the only possible case for
8717                                  * YCBCR444 usage is driving YCBCR420 output
8718                                  * with LSPCON, when pipe is configured for
8719                                  * YCBCR444 output and LSPCON takes care of
8720                                  * downsampling it.
8721                                  */
8722                                 pipe_config->lspcon_downsampling = true;
8723                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
8724                         }
8725                 }
8726         }
8727
8728         pipe_config->output_format = output;
8729 }
8730
8731 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
8732 {
8733         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8734         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8735         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8736         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8737         u32 tmp;
8738
8739         tmp = I915_READ(DSPCNTR(i9xx_plane));
8740
8741         if (tmp & DISPPLANE_GAMMA_ENABLE)
8742                 crtc_state->gamma_enable = true;
8743
8744         if (!HAS_GMCH(dev_priv) &&
8745             tmp & DISPPLANE_PIPE_CSC_ENABLE)
8746                 crtc_state->csc_enable = true;
8747 }
8748
8749 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8750                                  struct intel_crtc_state *pipe_config)
8751 {
8752         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8753         enum intel_display_power_domain power_domain;
8754         intel_wakeref_t wakeref;
8755         u32 tmp;
8756         bool ret;
8757
8758         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8759         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
8760         if (!wakeref)
8761                 return false;
8762
8763         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
8764         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8765         pipe_config->shared_dpll = NULL;
8766
8767         ret = false;
8768
8769         tmp = I915_READ(PIPECONF(crtc->pipe));
8770         if (!(tmp & PIPECONF_ENABLE))
8771                 goto out;
8772
8773         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8774             IS_CHERRYVIEW(dev_priv)) {
8775                 switch (tmp & PIPECONF_BPC_MASK) {
8776                 case PIPECONF_6BPC:
8777                         pipe_config->pipe_bpp = 18;
8778                         break;
8779                 case PIPECONF_8BPC:
8780                         pipe_config->pipe_bpp = 24;
8781                         break;
8782                 case PIPECONF_10BPC:
8783                         pipe_config->pipe_bpp = 30;
8784                         break;
8785                 default:
8786                         break;
8787                 }
8788         }
8789
8790         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8791             (tmp & PIPECONF_COLOR_RANGE_SELECT))
8792                 pipe_config->limited_color_range = true;
8793
8794         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
8795                 PIPECONF_GAMMA_MODE_SHIFT;
8796
8797         if (IS_CHERRYVIEW(dev_priv))
8798                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
8799
8800         i9xx_get_pipe_color_config(pipe_config);
8801         intel_color_get_config(pipe_config);
8802
8803         if (INTEL_GEN(dev_priv) < 4)
8804                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8805
8806         intel_get_pipe_timings(crtc, pipe_config);
8807         intel_get_pipe_src_size(crtc, pipe_config);
8808
8809         i9xx_get_pfit_config(crtc, pipe_config);
8810
8811         if (INTEL_GEN(dev_priv) >= 4) {
8812                 /* No way to read it out on pipes B and C */
8813                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8814                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8815                 else
8816                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8817                 pipe_config->pixel_multiplier =
8818                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8819                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8820                 pipe_config->dpll_hw_state.dpll_md = tmp;
8821         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8822                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8823                 tmp = I915_READ(DPLL(crtc->pipe));
8824                 pipe_config->pixel_multiplier =
8825                         ((tmp & SDVO_MULTIPLIER_MASK)
8826                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8827         } else {
8828                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8829                  * port and will be fixed up in the encoder->get_config
8830                  * function. */
8831                 pipe_config->pixel_multiplier = 1;
8832         }
8833         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8834         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8835                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8836                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8837         } else {
8838                 /* Mask out read-only status bits. */
8839                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8840                                                      DPLL_PORTC_READY_MASK |
8841                                                      DPLL_PORTB_READY_MASK);
8842         }
8843
8844         if (IS_CHERRYVIEW(dev_priv))
8845                 chv_crtc_clock_get(crtc, pipe_config);
8846         else if (IS_VALLEYVIEW(dev_priv))
8847                 vlv_crtc_clock_get(crtc, pipe_config);
8848         else
8849                 i9xx_crtc_clock_get(crtc, pipe_config);
8850
8851         /*
8852          * Normally the dotclock is filled in by the encoder .get_config()
8853          * but in case the pipe is enabled w/o any ports we need a sane
8854          * default.
8855          */
8856         pipe_config->base.adjusted_mode.crtc_clock =
8857                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8858
8859         ret = true;
8860
8861 out:
8862         intel_display_power_put(dev_priv, power_domain, wakeref);
8863
8864         return ret;
8865 }
8866
8867 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8868 {
8869         struct intel_encoder *encoder;
8870         int i;
8871         u32 val, final;
8872         bool has_lvds = false;
8873         bool has_cpu_edp = false;
8874         bool has_panel = false;
8875         bool has_ck505 = false;
8876         bool can_ssc = false;
8877         bool using_ssc_source = false;
8878
8879         /* We need to take the global config into account */
8880         for_each_intel_encoder(&dev_priv->drm, encoder) {
8881                 switch (encoder->type) {
8882                 case INTEL_OUTPUT_LVDS:
8883                         has_panel = true;
8884                         has_lvds = true;
8885                         break;
8886                 case INTEL_OUTPUT_EDP:
8887                         has_panel = true;
8888                         if (encoder->port == PORT_A)
8889                                 has_cpu_edp = true;
8890                         break;
8891                 default:
8892                         break;
8893                 }
8894         }
8895
8896         if (HAS_PCH_IBX(dev_priv)) {
8897                 has_ck505 = dev_priv->vbt.display_clock_mode;
8898                 can_ssc = has_ck505;
8899         } else {
8900                 has_ck505 = false;
8901                 can_ssc = true;
8902         }
8903
8904         /* Check if any DPLLs are using the SSC source */
8905         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8906                 u32 temp = I915_READ(PCH_DPLL(i));
8907
8908                 if (!(temp & DPLL_VCO_ENABLE))
8909                         continue;
8910
8911                 if ((temp & PLL_REF_INPUT_MASK) ==
8912                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8913                         using_ssc_source = true;
8914                         break;
8915                 }
8916         }
8917
8918         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8919                       has_panel, has_lvds, has_ck505, using_ssc_source);
8920
8921         /* Ironlake: try to setup display ref clock before DPLL
8922          * enabling. This is only under driver's control after
8923          * PCH B stepping, previous chipset stepping should be
8924          * ignoring this setting.
8925          */
8926         val = I915_READ(PCH_DREF_CONTROL);
8927
8928         /* As we must carefully and slowly disable/enable each source in turn,
8929          * compute the final state we want first and check if we need to
8930          * make any changes at all.
8931          */
8932         final = val;
8933         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8934         if (has_ck505)
8935                 final |= DREF_NONSPREAD_CK505_ENABLE;
8936         else
8937                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8938
8939         final &= ~DREF_SSC_SOURCE_MASK;
8940         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8941         final &= ~DREF_SSC1_ENABLE;
8942
8943         if (has_panel) {
8944                 final |= DREF_SSC_SOURCE_ENABLE;
8945
8946                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8947                         final |= DREF_SSC1_ENABLE;
8948
8949                 if (has_cpu_edp) {
8950                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8951                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8952                         else
8953                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8954                 } else
8955                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8956         } else if (using_ssc_source) {
8957                 final |= DREF_SSC_SOURCE_ENABLE;
8958                 final |= DREF_SSC1_ENABLE;
8959         }
8960
8961         if (final == val)
8962                 return;
8963
8964         /* Always enable nonspread source */
8965         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8966
8967         if (has_ck505)
8968                 val |= DREF_NONSPREAD_CK505_ENABLE;
8969         else
8970                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8971
8972         if (has_panel) {
8973                 val &= ~DREF_SSC_SOURCE_MASK;
8974                 val |= DREF_SSC_SOURCE_ENABLE;
8975
8976                 /* SSC must be turned on before enabling the CPU output  */
8977                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8978                         DRM_DEBUG_KMS("Using SSC on panel\n");
8979                         val |= DREF_SSC1_ENABLE;
8980                 } else
8981                         val &= ~DREF_SSC1_ENABLE;
8982
8983                 /* Get SSC going before enabling the outputs */
8984                 I915_WRITE(PCH_DREF_CONTROL, val);
8985                 POSTING_READ(PCH_DREF_CONTROL);
8986                 udelay(200);
8987
8988                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8989
8990                 /* Enable CPU source on CPU attached eDP */
8991                 if (has_cpu_edp) {
8992                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8993                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8994                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8995                         } else
8996                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8997                 } else
8998                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8999
9000                 I915_WRITE(PCH_DREF_CONTROL, val);
9001                 POSTING_READ(PCH_DREF_CONTROL);
9002                 udelay(200);
9003         } else {
9004                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9005
9006                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9007
9008                 /* Turn off CPU output */
9009                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9010
9011                 I915_WRITE(PCH_DREF_CONTROL, val);
9012                 POSTING_READ(PCH_DREF_CONTROL);
9013                 udelay(200);
9014
9015                 if (!using_ssc_source) {
9016                         DRM_DEBUG_KMS("Disabling SSC source\n");
9017
9018                         /* Turn off the SSC source */
9019                         val &= ~DREF_SSC_SOURCE_MASK;
9020                         val |= DREF_SSC_SOURCE_DISABLE;
9021
9022                         /* Turn off SSC1 */
9023                         val &= ~DREF_SSC1_ENABLE;
9024
9025                         I915_WRITE(PCH_DREF_CONTROL, val);
9026                         POSTING_READ(PCH_DREF_CONTROL);
9027                         udelay(200);
9028                 }
9029         }
9030
9031         BUG_ON(val != final);
9032 }
9033
9034 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9035 {
9036         u32 tmp;
9037
9038         tmp = I915_READ(SOUTH_CHICKEN2);
9039         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9040         I915_WRITE(SOUTH_CHICKEN2, tmp);
9041
9042         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9043                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9044                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9045
9046         tmp = I915_READ(SOUTH_CHICKEN2);
9047         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9048         I915_WRITE(SOUTH_CHICKEN2, tmp);
9049
9050         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9051                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9052                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9053 }
9054
9055 /* WaMPhyProgramming:hsw */
9056 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9057 {
9058         u32 tmp;
9059
9060         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9061         tmp &= ~(0xFF << 24);
9062         tmp |= (0x12 << 24);
9063         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9064
9065         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9066         tmp |= (1 << 11);
9067         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9068
9069         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9070         tmp |= (1 << 11);
9071         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9072
9073         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9074         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9075         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9076
9077         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9078         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9079         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9080
9081         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9082         tmp &= ~(7 << 13);
9083         tmp |= (5 << 13);
9084         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9085
9086         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9087         tmp &= ~(7 << 13);
9088         tmp |= (5 << 13);
9089         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9090
9091         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9092         tmp &= ~0xFF;
9093         tmp |= 0x1C;
9094         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9095
9096         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9097         tmp &= ~0xFF;
9098         tmp |= 0x1C;
9099         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9100
9101         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9102         tmp &= ~(0xFF << 16);
9103         tmp |= (0x1C << 16);
9104         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9105
9106         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9107         tmp &= ~(0xFF << 16);
9108         tmp |= (0x1C << 16);
9109         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9110
9111         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9112         tmp |= (1 << 27);
9113         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9114
9115         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9116         tmp |= (1 << 27);
9117         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9118
9119         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9120         tmp &= ~(0xF << 28);
9121         tmp |= (4 << 28);
9122         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9123
9124         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9125         tmp &= ~(0xF << 28);
9126         tmp |= (4 << 28);
9127         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9128 }
9129
9130 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9131  * Programming" based on the parameters passed:
9132  * - Sequence to enable CLKOUT_DP
9133  * - Sequence to enable CLKOUT_DP without spread
9134  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9135  */
9136 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9137                                  bool with_spread, bool with_fdi)
9138 {
9139         u32 reg, tmp;
9140
9141         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9142                 with_spread = true;
9143         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9144             with_fdi, "LP PCH doesn't have FDI\n"))
9145                 with_fdi = false;
9146
9147         mutex_lock(&dev_priv->sb_lock);
9148
9149         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9150         tmp &= ~SBI_SSCCTL_DISABLE;
9151         tmp |= SBI_SSCCTL_PATHALT;
9152         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9153
9154         udelay(24);
9155
9156         if (with_spread) {
9157                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9158                 tmp &= ~SBI_SSCCTL_PATHALT;
9159                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9160
9161                 if (with_fdi) {
9162                         lpt_reset_fdi_mphy(dev_priv);
9163                         lpt_program_fdi_mphy(dev_priv);
9164                 }
9165         }
9166
9167         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9168         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9169         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9170         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9171
9172         mutex_unlock(&dev_priv->sb_lock);
9173 }
9174
9175 /* Sequence to disable CLKOUT_DP */
9176 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9177 {
9178         u32 reg, tmp;
9179
9180         mutex_lock(&dev_priv->sb_lock);
9181
9182         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9183         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9184         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9185         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9186
9187         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9188         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9189                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9190                         tmp |= SBI_SSCCTL_PATHALT;
9191                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9192                         udelay(32);
9193                 }
9194                 tmp |= SBI_SSCCTL_DISABLE;
9195                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9196         }
9197
9198         mutex_unlock(&dev_priv->sb_lock);
9199 }
9200
9201 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9202
9203 static const u16 sscdivintphase[] = {
9204         [BEND_IDX( 50)] = 0x3B23,
9205         [BEND_IDX( 45)] = 0x3B23,
9206         [BEND_IDX( 40)] = 0x3C23,
9207         [BEND_IDX( 35)] = 0x3C23,
9208         [BEND_IDX( 30)] = 0x3D23,
9209         [BEND_IDX( 25)] = 0x3D23,
9210         [BEND_IDX( 20)] = 0x3E23,
9211         [BEND_IDX( 15)] = 0x3E23,
9212         [BEND_IDX( 10)] = 0x3F23,
9213         [BEND_IDX(  5)] = 0x3F23,
9214         [BEND_IDX(  0)] = 0x0025,
9215         [BEND_IDX( -5)] = 0x0025,
9216         [BEND_IDX(-10)] = 0x0125,
9217         [BEND_IDX(-15)] = 0x0125,
9218         [BEND_IDX(-20)] = 0x0225,
9219         [BEND_IDX(-25)] = 0x0225,
9220         [BEND_IDX(-30)] = 0x0325,
9221         [BEND_IDX(-35)] = 0x0325,
9222         [BEND_IDX(-40)] = 0x0425,
9223         [BEND_IDX(-45)] = 0x0425,
9224         [BEND_IDX(-50)] = 0x0525,
9225 };
9226
9227 /*
9228  * Bend CLKOUT_DP
9229  * steps -50 to 50 inclusive, in steps of 5
9230  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9231  * change in clock period = -(steps / 10) * 5.787 ps
9232  */
9233 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9234 {
9235         u32 tmp;
9236         int idx = BEND_IDX(steps);
9237
9238         if (WARN_ON(steps % 5 != 0))
9239                 return;
9240
9241         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9242                 return;
9243
9244         mutex_lock(&dev_priv->sb_lock);
9245
9246         if (steps % 10 != 0)
9247                 tmp = 0xAAAAAAAB;
9248         else
9249                 tmp = 0x00000000;
9250         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9251
9252         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9253         tmp &= 0xffff0000;
9254         tmp |= sscdivintphase[idx];
9255         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9256
9257         mutex_unlock(&dev_priv->sb_lock);
9258 }
9259
9260 #undef BEND_IDX
9261
9262 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9263 {
9264         u32 fuse_strap = I915_READ(FUSE_STRAP);
9265         u32 ctl = I915_READ(SPLL_CTL);
9266
9267         if ((ctl & SPLL_PLL_ENABLE) == 0)
9268                 return false;
9269
9270         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9271             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9272                 return true;
9273
9274         if (IS_BROADWELL(dev_priv) &&
9275             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9276                 return true;
9277
9278         return false;
9279 }
9280
9281 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9282                                enum intel_dpll_id id)
9283 {
9284         u32 fuse_strap = I915_READ(FUSE_STRAP);
9285         u32 ctl = I915_READ(WRPLL_CTL(id));
9286
9287         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9288                 return false;
9289
9290         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9291                 return true;
9292
9293         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9294             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9295             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9296                 return true;
9297
9298         return false;
9299 }
9300
9301 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9302 {
9303         struct intel_encoder *encoder;
9304         bool pch_ssc_in_use = false;
9305         bool has_fdi = false;
9306
9307         for_each_intel_encoder(&dev_priv->drm, encoder) {
9308                 switch (encoder->type) {
9309                 case INTEL_OUTPUT_ANALOG:
9310                         has_fdi = true;
9311                         break;
9312                 default:
9313                         break;
9314                 }
9315         }
9316
9317         /*
9318          * The BIOS may have decided to use the PCH SSC
9319          * reference so we must not disable it until the
9320          * relevant PLLs have stopped relying on it. We'll
9321          * just leave the PCH SSC reference enabled in case
9322          * any active PLL is using it. It will get disabled
9323          * after runtime suspend if we don't have FDI.
9324          *
9325          * TODO: Move the whole reference clock handling
9326          * to the modeset sequence proper so that we can
9327          * actually enable/disable/reconfigure these things
9328          * safely. To do that we need to introduce a real
9329          * clock hierarchy. That would also allow us to do
9330          * clock bending finally.
9331          */
9332         if (spll_uses_pch_ssc(dev_priv)) {
9333                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9334                 pch_ssc_in_use = true;
9335         }
9336
9337         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9338                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9339                 pch_ssc_in_use = true;
9340         }
9341
9342         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9343                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9344                 pch_ssc_in_use = true;
9345         }
9346
9347         if (pch_ssc_in_use)
9348                 return;
9349
9350         if (has_fdi) {
9351                 lpt_bend_clkout_dp(dev_priv, 0);
9352                 lpt_enable_clkout_dp(dev_priv, true, true);
9353         } else {
9354                 lpt_disable_clkout_dp(dev_priv);
9355         }
9356 }
9357
9358 /*
9359  * Initialize reference clocks when the driver loads
9360  */
9361 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9362 {
9363         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9364                 ironlake_init_pch_refclk(dev_priv);
9365         else if (HAS_PCH_LPT(dev_priv))
9366                 lpt_init_pch_refclk(dev_priv);
9367 }
9368
9369 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9370 {
9371         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9372         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9373         enum pipe pipe = crtc->pipe;
9374         u32 val;
9375
9376         val = 0;
9377
9378         switch (crtc_state->pipe_bpp) {
9379         case 18:
9380                 val |= PIPECONF_6BPC;
9381                 break;
9382         case 24:
9383                 val |= PIPECONF_8BPC;
9384                 break;
9385         case 30:
9386                 val |= PIPECONF_10BPC;
9387                 break;
9388         case 36:
9389                 val |= PIPECONF_12BPC;
9390                 break;
9391         default:
9392                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9393                 BUG();
9394         }
9395
9396         if (crtc_state->dither)
9397                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9398
9399         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9400                 val |= PIPECONF_INTERLACED_ILK;
9401         else
9402                 val |= PIPECONF_PROGRESSIVE;
9403
9404         if (crtc_state->limited_color_range)
9405                 val |= PIPECONF_COLOR_RANGE_SELECT;
9406
9407         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9408
9409         I915_WRITE(PIPECONF(pipe), val);
9410         POSTING_READ(PIPECONF(pipe));
9411 }
9412
9413 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9414 {
9415         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9417         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9418         u32 val = 0;
9419
9420         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9421                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9422
9423         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9424                 val |= PIPECONF_INTERLACED_ILK;
9425         else
9426                 val |= PIPECONF_PROGRESSIVE;
9427
9428         I915_WRITE(PIPECONF(cpu_transcoder), val);
9429         POSTING_READ(PIPECONF(cpu_transcoder));
9430 }
9431
9432 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9433 {
9434         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9435         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9436         u32 val = 0;
9437
9438         switch (crtc_state->pipe_bpp) {
9439         case 18:
9440                 val |= PIPEMISC_DITHER_6_BPC;
9441                 break;
9442         case 24:
9443                 val |= PIPEMISC_DITHER_8_BPC;
9444                 break;
9445         case 30:
9446                 val |= PIPEMISC_DITHER_10_BPC;
9447                 break;
9448         case 36:
9449                 val |= PIPEMISC_DITHER_12_BPC;
9450                 break;
9451         default:
9452                 MISSING_CASE(crtc_state->pipe_bpp);
9453                 break;
9454         }
9455
9456         if (crtc_state->dither)
9457                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9458
9459         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9460             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9461                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9462
9463         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9464                 val |= PIPEMISC_YUV420_ENABLE |
9465                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9466
9467         if (INTEL_GEN(dev_priv) >= 11 &&
9468             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9469                                            BIT(PLANE_CURSOR))) == 0)
9470                 val |= PIPEMISC_HDR_MODE_PRECISION;
9471
9472         I915_WRITE(PIPEMISC(crtc->pipe), val);
9473 }
9474
9475 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9476 {
9477         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9478         u32 tmp;
9479
9480         tmp = I915_READ(PIPEMISC(crtc->pipe));
9481
9482         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9483         case PIPEMISC_DITHER_6_BPC:
9484                 return 18;
9485         case PIPEMISC_DITHER_8_BPC:
9486                 return 24;
9487         case PIPEMISC_DITHER_10_BPC:
9488                 return 30;
9489         case PIPEMISC_DITHER_12_BPC:
9490                 return 36;
9491         default:
9492                 MISSING_CASE(tmp);
9493                 return 0;
9494         }
9495 }
9496
9497 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9498 {
9499         /*
9500          * Account for spread spectrum to avoid
9501          * oversubscribing the link. Max center spread
9502          * is 2.5%; use 5% for safety's sake.
9503          */
9504         u32 bps = target_clock * bpp * 21 / 20;
9505         return DIV_ROUND_UP(bps, link_bw * 8);
9506 }
9507
9508 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9509 {
9510         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9511 }
9512
9513 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9514                                   struct intel_crtc_state *crtc_state,
9515                                   struct dpll *reduced_clock)
9516 {
9517         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9518         u32 dpll, fp, fp2;
9519         int factor;
9520
9521         /* Enable autotuning of the PLL clock (if permissible) */
9522         factor = 21;
9523         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9524                 if ((intel_panel_use_ssc(dev_priv) &&
9525                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9526                     (HAS_PCH_IBX(dev_priv) &&
9527                      intel_is_dual_link_lvds(dev_priv)))
9528                         factor = 25;
9529         } else if (crtc_state->sdvo_tv_clock) {
9530                 factor = 20;
9531         }
9532
9533         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9534
9535         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9536                 fp |= FP_CB_TUNE;
9537
9538         if (reduced_clock) {
9539                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9540
9541                 if (reduced_clock->m < factor * reduced_clock->n)
9542                         fp2 |= FP_CB_TUNE;
9543         } else {
9544                 fp2 = fp;
9545         }
9546
9547         dpll = 0;
9548
9549         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9550                 dpll |= DPLLB_MODE_LVDS;
9551         else
9552                 dpll |= DPLLB_MODE_DAC_SERIAL;
9553
9554         dpll |= (crtc_state->pixel_multiplier - 1)
9555                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9556
9557         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9558             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9559                 dpll |= DPLL_SDVO_HIGH_SPEED;
9560
9561         if (intel_crtc_has_dp_encoder(crtc_state))
9562                 dpll |= DPLL_SDVO_HIGH_SPEED;
9563
9564         /*
9565          * The high speed IO clock is only really required for
9566          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9567          * possible to share the DPLL between CRT and HDMI. Enabling
9568          * the clock needlessly does no real harm, except use up a
9569          * bit of power potentially.
9570          *
9571          * We'll limit this to IVB with 3 pipes, since it has only two
9572          * DPLLs and so DPLL sharing is the only way to get three pipes
9573          * driving PCH ports at the same time. On SNB we could do this,
9574          * and potentially avoid enabling the second DPLL, but it's not
9575          * clear if it''s a win or loss power wise. No point in doing
9576          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9577          */
9578         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
9579             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9580                 dpll |= DPLL_SDVO_HIGH_SPEED;
9581
9582         /* compute bitmask from p1 value */
9583         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9584         /* also FPA1 */
9585         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9586
9587         switch (crtc_state->dpll.p2) {
9588         case 5:
9589                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9590                 break;
9591         case 7:
9592                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9593                 break;
9594         case 10:
9595                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9596                 break;
9597         case 14:
9598                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9599                 break;
9600         }
9601
9602         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9603             intel_panel_use_ssc(dev_priv))
9604                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9605         else
9606                 dpll |= PLL_REF_INPUT_DREFCLK;
9607
9608         dpll |= DPLL_VCO_ENABLE;
9609
9610         crtc_state->dpll_hw_state.dpll = dpll;
9611         crtc_state->dpll_hw_state.fp0 = fp;
9612         crtc_state->dpll_hw_state.fp1 = fp2;
9613 }
9614
9615 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9616                                        struct intel_crtc_state *crtc_state)
9617 {
9618         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9619         struct intel_atomic_state *state =
9620                 to_intel_atomic_state(crtc_state->base.state);
9621         const struct intel_limit *limit;
9622         int refclk = 120000;
9623
9624         memset(&crtc_state->dpll_hw_state, 0,
9625                sizeof(crtc_state->dpll_hw_state));
9626
9627         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9628         if (!crtc_state->has_pch_encoder)
9629                 return 0;
9630
9631         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9632                 if (intel_panel_use_ssc(dev_priv)) {
9633                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9634                                       dev_priv->vbt.lvds_ssc_freq);
9635                         refclk = dev_priv->vbt.lvds_ssc_freq;
9636                 }
9637
9638                 if (intel_is_dual_link_lvds(dev_priv)) {
9639                         if (refclk == 100000)
9640                                 limit = &intel_limits_ironlake_dual_lvds_100m;
9641                         else
9642                                 limit = &intel_limits_ironlake_dual_lvds;
9643                 } else {
9644                         if (refclk == 100000)
9645                                 limit = &intel_limits_ironlake_single_lvds_100m;
9646                         else
9647                                 limit = &intel_limits_ironlake_single_lvds;
9648                 }
9649         } else {
9650                 limit = &intel_limits_ironlake_dac;
9651         }
9652
9653         if (!crtc_state->clock_set &&
9654             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9655                                 refclk, NULL, &crtc_state->dpll)) {
9656                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
9657                 return -EINVAL;
9658         }
9659
9660         ironlake_compute_dpll(crtc, crtc_state, NULL);
9661
9662         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
9663                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9664                               pipe_name(crtc->pipe));
9665                 return -EINVAL;
9666         }
9667
9668         return 0;
9669 }
9670
9671 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
9672                                          struct intel_link_m_n *m_n)
9673 {
9674         struct drm_device *dev = crtc->base.dev;
9675         struct drm_i915_private *dev_priv = to_i915(dev);
9676         enum pipe pipe = crtc->pipe;
9677
9678         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9679         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9680         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9681                 & ~TU_SIZE_MASK;
9682         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9683         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9684                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9685 }
9686
9687 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9688                                          enum transcoder transcoder,
9689                                          struct intel_link_m_n *m_n,
9690                                          struct intel_link_m_n *m2_n2)
9691 {
9692         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9693         enum pipe pipe = crtc->pipe;
9694
9695         if (INTEL_GEN(dev_priv) >= 5) {
9696                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9697                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9698                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9699                         & ~TU_SIZE_MASK;
9700                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9701                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9702                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9703
9704                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
9705                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9706                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9707                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9708                                         & ~TU_SIZE_MASK;
9709                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9710                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9711                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9712                 }
9713         } else {
9714                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9715                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9716                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9717                         & ~TU_SIZE_MASK;
9718                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9719                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9720                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9721         }
9722 }
9723
9724 void intel_dp_get_m_n(struct intel_crtc *crtc,
9725                       struct intel_crtc_state *pipe_config)
9726 {
9727         if (pipe_config->has_pch_encoder)
9728                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9729         else
9730                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9731                                              &pipe_config->dp_m_n,
9732                                              &pipe_config->dp_m2_n2);
9733 }
9734
9735 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9736                                         struct intel_crtc_state *pipe_config)
9737 {
9738         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9739                                      &pipe_config->fdi_m_n, NULL);
9740 }
9741
9742 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9743                                     struct intel_crtc_state *pipe_config)
9744 {
9745         struct drm_device *dev = crtc->base.dev;
9746         struct drm_i915_private *dev_priv = to_i915(dev);
9747         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9748         u32 ps_ctrl = 0;
9749         int id = -1;
9750         int i;
9751
9752         /* find scaler attached to this pipe */
9753         for (i = 0; i < crtc->num_scalers; i++) {
9754                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9755                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9756                         id = i;
9757                         pipe_config->pch_pfit.enabled = true;
9758                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9759                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9760                         scaler_state->scalers[i].in_use = true;
9761                         break;
9762                 }
9763         }
9764
9765         scaler_state->scaler_id = id;
9766         if (id >= 0) {
9767                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9768         } else {
9769                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9770         }
9771 }
9772
9773 static void
9774 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9775                                  struct intel_initial_plane_config *plane_config)
9776 {
9777         struct drm_device *dev = crtc->base.dev;
9778         struct drm_i915_private *dev_priv = to_i915(dev);
9779         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9780         enum plane_id plane_id = plane->id;
9781         enum pipe pipe;
9782         u32 val, base, offset, stride_mult, tiling, alpha;
9783         int fourcc, pixel_format;
9784         unsigned int aligned_height;
9785         struct drm_framebuffer *fb;
9786         struct intel_framebuffer *intel_fb;
9787
9788         if (!plane->get_hw_state(plane, &pipe))
9789                 return;
9790
9791         WARN_ON(pipe != crtc->pipe);
9792
9793         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9794         if (!intel_fb) {
9795                 DRM_DEBUG_KMS("failed to alloc fb\n");
9796                 return;
9797         }
9798
9799         fb = &intel_fb->base;
9800
9801         fb->dev = dev;
9802
9803         val = I915_READ(PLANE_CTL(pipe, plane_id));
9804
9805         if (INTEL_GEN(dev_priv) >= 11)
9806                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
9807         else
9808                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
9809
9810         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
9811                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
9812                 alpha &= PLANE_COLOR_ALPHA_MASK;
9813         } else {
9814                 alpha = val & PLANE_CTL_ALPHA_MASK;
9815         }
9816
9817         fourcc = skl_format_to_fourcc(pixel_format,
9818                                       val & PLANE_CTL_ORDER_RGBX, alpha);
9819         fb->format = drm_format_info(fourcc);
9820
9821         tiling = val & PLANE_CTL_TILED_MASK;
9822         switch (tiling) {
9823         case PLANE_CTL_TILED_LINEAR:
9824                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
9825                 break;
9826         case PLANE_CTL_TILED_X:
9827                 plane_config->tiling = I915_TILING_X;
9828                 fb->modifier = I915_FORMAT_MOD_X_TILED;
9829                 break;
9830         case PLANE_CTL_TILED_Y:
9831                 plane_config->tiling = I915_TILING_Y;
9832                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9833                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
9834                 else
9835                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
9836                 break;
9837         case PLANE_CTL_TILED_YF:
9838                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9839                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
9840                 else
9841                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
9842                 break;
9843         default:
9844                 MISSING_CASE(tiling);
9845                 goto error;
9846         }
9847
9848         /*
9849          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
9850          * while i915 HW rotation is clockwise, thats why this swapping.
9851          */
9852         switch (val & PLANE_CTL_ROTATE_MASK) {
9853         case PLANE_CTL_ROTATE_0:
9854                 plane_config->rotation = DRM_MODE_ROTATE_0;
9855                 break;
9856         case PLANE_CTL_ROTATE_90:
9857                 plane_config->rotation = DRM_MODE_ROTATE_270;
9858                 break;
9859         case PLANE_CTL_ROTATE_180:
9860                 plane_config->rotation = DRM_MODE_ROTATE_180;
9861                 break;
9862         case PLANE_CTL_ROTATE_270:
9863                 plane_config->rotation = DRM_MODE_ROTATE_90;
9864                 break;
9865         }
9866
9867         if (INTEL_GEN(dev_priv) >= 10 &&
9868             val & PLANE_CTL_FLIP_HORIZONTAL)
9869                 plane_config->rotation |= DRM_MODE_REFLECT_X;
9870
9871         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
9872         plane_config->base = base;
9873
9874         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
9875
9876         val = I915_READ(PLANE_SIZE(pipe, plane_id));
9877         fb->height = ((val >> 16) & 0xfff) + 1;
9878         fb->width = ((val >> 0) & 0x1fff) + 1;
9879
9880         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
9881         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
9882         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9883
9884         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9885
9886         plane_config->size = fb->pitches[0] * aligned_height;
9887
9888         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9889                       crtc->base.name, plane->base.name, fb->width, fb->height,
9890                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9891                       plane_config->size);
9892
9893         plane_config->fb = intel_fb;
9894         return;
9895
9896 error:
9897         kfree(intel_fb);
9898 }
9899
9900 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9901                                      struct intel_crtc_state *pipe_config)
9902 {
9903         struct drm_device *dev = crtc->base.dev;
9904         struct drm_i915_private *dev_priv = to_i915(dev);
9905         u32 tmp;
9906
9907         tmp = I915_READ(PF_CTL(crtc->pipe));
9908
9909         if (tmp & PF_ENABLE) {
9910                 pipe_config->pch_pfit.enabled = true;
9911                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9912                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9913
9914                 /* We currently do not free assignements of panel fitters on
9915                  * ivb/hsw (since we don't use the higher upscaling modes which
9916                  * differentiates them) so just WARN about this case for now. */
9917                 if (IS_GEN(dev_priv, 7)) {
9918                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9919                                 PF_PIPE_SEL_IVB(crtc->pipe));
9920                 }
9921         }
9922 }
9923
9924 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9925                                      struct intel_crtc_state *pipe_config)
9926 {
9927         struct drm_device *dev = crtc->base.dev;
9928         struct drm_i915_private *dev_priv = to_i915(dev);
9929         enum intel_display_power_domain power_domain;
9930         intel_wakeref_t wakeref;
9931         u32 tmp;
9932         bool ret;
9933
9934         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9935         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9936         if (!wakeref)
9937                 return false;
9938
9939         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9940         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9941         pipe_config->shared_dpll = NULL;
9942
9943         ret = false;
9944         tmp = I915_READ(PIPECONF(crtc->pipe));
9945         if (!(tmp & PIPECONF_ENABLE))
9946                 goto out;
9947
9948         switch (tmp & PIPECONF_BPC_MASK) {
9949         case PIPECONF_6BPC:
9950                 pipe_config->pipe_bpp = 18;
9951                 break;
9952         case PIPECONF_8BPC:
9953                 pipe_config->pipe_bpp = 24;
9954                 break;
9955         case PIPECONF_10BPC:
9956                 pipe_config->pipe_bpp = 30;
9957                 break;
9958         case PIPECONF_12BPC:
9959                 pipe_config->pipe_bpp = 36;
9960                 break;
9961         default:
9962                 break;
9963         }
9964
9965         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9966                 pipe_config->limited_color_range = true;
9967
9968         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
9969                 PIPECONF_GAMMA_MODE_SHIFT;
9970
9971         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
9972
9973         i9xx_get_pipe_color_config(pipe_config);
9974         intel_color_get_config(pipe_config);
9975
9976         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9977                 struct intel_shared_dpll *pll;
9978                 enum intel_dpll_id pll_id;
9979
9980                 pipe_config->has_pch_encoder = true;
9981
9982                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9983                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9984                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9985
9986                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9987
9988                 if (HAS_PCH_IBX(dev_priv)) {
9989                         /*
9990                          * The pipe->pch transcoder and pch transcoder->pll
9991                          * mapping is fixed.
9992                          */
9993                         pll_id = (enum intel_dpll_id) crtc->pipe;
9994                 } else {
9995                         tmp = I915_READ(PCH_DPLL_SEL);
9996                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9997                                 pll_id = DPLL_ID_PCH_PLL_B;
9998                         else
9999                                 pll_id= DPLL_ID_PCH_PLL_A;
10000                 }
10001
10002                 pipe_config->shared_dpll =
10003                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10004                 pll = pipe_config->shared_dpll;
10005
10006                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10007                                                 &pipe_config->dpll_hw_state));
10008
10009                 tmp = pipe_config->dpll_hw_state.dpll;
10010                 pipe_config->pixel_multiplier =
10011                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10012                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10013
10014                 ironlake_pch_clock_get(crtc, pipe_config);
10015         } else {
10016                 pipe_config->pixel_multiplier = 1;
10017         }
10018
10019         intel_get_pipe_timings(crtc, pipe_config);
10020         intel_get_pipe_src_size(crtc, pipe_config);
10021
10022         ironlake_get_pfit_config(crtc, pipe_config);
10023
10024         ret = true;
10025
10026 out:
10027         intel_display_power_put(dev_priv, power_domain, wakeref);
10028
10029         return ret;
10030 }
10031 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10032                                       struct intel_crtc_state *crtc_state)
10033 {
10034         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10035         struct intel_atomic_state *state =
10036                 to_intel_atomic_state(crtc_state->base.state);
10037
10038         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10039             INTEL_GEN(dev_priv) >= 11) {
10040                 struct intel_encoder *encoder =
10041                         intel_get_crtc_new_encoder(state, crtc_state);
10042
10043                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10044                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10045                                       pipe_name(crtc->pipe));
10046                         return -EINVAL;
10047                 }
10048         }
10049
10050         return 0;
10051 }
10052
10053 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10054                                    enum port port,
10055                                    struct intel_crtc_state *pipe_config)
10056 {
10057         enum intel_dpll_id id;
10058         u32 temp;
10059
10060         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10061         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10062
10063         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10064                 return;
10065
10066         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10067 }
10068
10069 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10070                                 enum port port,
10071                                 struct intel_crtc_state *pipe_config)
10072 {
10073         enum phy phy = intel_port_to_phy(dev_priv, port);
10074         enum icl_port_dpll_id port_dpll_id;
10075         enum intel_dpll_id id;
10076         u32 temp;
10077
10078         if (intel_phy_is_combo(dev_priv, phy)) {
10079                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10080                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10081                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10082                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10083         } else if (intel_phy_is_tc(dev_priv, phy)) {
10084                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10085
10086                 if (clk_sel == DDI_CLK_SEL_MG) {
10087                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10088                                                                     port));
10089                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10090                 } else {
10091                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10092                         id = DPLL_ID_ICL_TBTPLL;
10093                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10094                 }
10095         } else {
10096                 WARN(1, "Invalid port %x\n", port);
10097                 return;
10098         }
10099
10100         pipe_config->icl_port_dplls[port_dpll_id].pll =
10101                 intel_get_shared_dpll_by_id(dev_priv, id);
10102
10103         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10104 }
10105
10106 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10107                                 enum port port,
10108                                 struct intel_crtc_state *pipe_config)
10109 {
10110         enum intel_dpll_id id;
10111
10112         switch (port) {
10113         case PORT_A:
10114                 id = DPLL_ID_SKL_DPLL0;
10115                 break;
10116         case PORT_B:
10117                 id = DPLL_ID_SKL_DPLL1;
10118                 break;
10119         case PORT_C:
10120                 id = DPLL_ID_SKL_DPLL2;
10121                 break;
10122         default:
10123                 DRM_ERROR("Incorrect port type\n");
10124                 return;
10125         }
10126
10127         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10128 }
10129
10130 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10131                                 enum port port,
10132                                 struct intel_crtc_state *pipe_config)
10133 {
10134         enum intel_dpll_id id;
10135         u32 temp;
10136
10137         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10138         id = temp >> (port * 3 + 1);
10139
10140         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10141                 return;
10142
10143         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10144 }
10145
10146 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10147                                 enum port port,
10148                                 struct intel_crtc_state *pipe_config)
10149 {
10150         enum intel_dpll_id id;
10151         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10152
10153         switch (ddi_pll_sel) {
10154         case PORT_CLK_SEL_WRPLL1:
10155                 id = DPLL_ID_WRPLL1;
10156                 break;
10157         case PORT_CLK_SEL_WRPLL2:
10158                 id = DPLL_ID_WRPLL2;
10159                 break;
10160         case PORT_CLK_SEL_SPLL:
10161                 id = DPLL_ID_SPLL;
10162                 break;
10163         case PORT_CLK_SEL_LCPLL_810:
10164                 id = DPLL_ID_LCPLL_810;
10165                 break;
10166         case PORT_CLK_SEL_LCPLL_1350:
10167                 id = DPLL_ID_LCPLL_1350;
10168                 break;
10169         case PORT_CLK_SEL_LCPLL_2700:
10170                 id = DPLL_ID_LCPLL_2700;
10171                 break;
10172         default:
10173                 MISSING_CASE(ddi_pll_sel);
10174                 /* fall through */
10175         case PORT_CLK_SEL_NONE:
10176                 return;
10177         }
10178
10179         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10180 }
10181
10182 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10183                                      struct intel_crtc_state *pipe_config,
10184                                      u64 *power_domain_mask,
10185                                      intel_wakeref_t *wakerefs)
10186 {
10187         struct drm_device *dev = crtc->base.dev;
10188         struct drm_i915_private *dev_priv = to_i915(dev);
10189         enum intel_display_power_domain power_domain;
10190         unsigned long panel_transcoder_mask = 0;
10191         unsigned long enabled_panel_transcoders = 0;
10192         enum transcoder panel_transcoder;
10193         intel_wakeref_t wf;
10194         u32 tmp;
10195
10196         if (INTEL_GEN(dev_priv) >= 11)
10197                 panel_transcoder_mask |=
10198                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10199
10200         if (HAS_TRANSCODER_EDP(dev_priv))
10201                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10202
10203         /*
10204          * The pipe->transcoder mapping is fixed with the exception of the eDP
10205          * and DSI transcoders handled below.
10206          */
10207         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10208
10209         /*
10210          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10211          * consistency and less surprising code; it's in always on power).
10212          */
10213         for_each_set_bit(panel_transcoder,
10214                          &panel_transcoder_mask,
10215                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10216                 bool force_thru = false;
10217                 enum pipe trans_pipe;
10218
10219                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10220                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10221                         continue;
10222
10223                 /*
10224                  * Log all enabled ones, only use the first one.
10225                  *
10226                  * FIXME: This won't work for two separate DSI displays.
10227                  */
10228                 enabled_panel_transcoders |= BIT(panel_transcoder);
10229                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10230                         continue;
10231
10232                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10233                 default:
10234                         WARN(1, "unknown pipe linked to transcoder %s\n",
10235                              transcoder_name(panel_transcoder));
10236                         /* fall through */
10237                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10238                         force_thru = true;
10239                         /* fall through */
10240                 case TRANS_DDI_EDP_INPUT_A_ON:
10241                         trans_pipe = PIPE_A;
10242                         break;
10243                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10244                         trans_pipe = PIPE_B;
10245                         break;
10246                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10247                         trans_pipe = PIPE_C;
10248                         break;
10249                 }
10250
10251                 if (trans_pipe == crtc->pipe) {
10252                         pipe_config->cpu_transcoder = panel_transcoder;
10253                         pipe_config->pch_pfit.force_thru = force_thru;
10254                 }
10255         }
10256
10257         /*
10258          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10259          */
10260         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10261                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10262
10263         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10264         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10265
10266         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10267         if (!wf)
10268                 return false;
10269
10270         wakerefs[power_domain] = wf;
10271         *power_domain_mask |= BIT_ULL(power_domain);
10272
10273         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10274
10275         return tmp & PIPECONF_ENABLE;
10276 }
10277
10278 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10279                                          struct intel_crtc_state *pipe_config,
10280                                          u64 *power_domain_mask,
10281                                          intel_wakeref_t *wakerefs)
10282 {
10283         struct drm_device *dev = crtc->base.dev;
10284         struct drm_i915_private *dev_priv = to_i915(dev);
10285         enum intel_display_power_domain power_domain;
10286         enum transcoder cpu_transcoder;
10287         intel_wakeref_t wf;
10288         enum port port;
10289         u32 tmp;
10290
10291         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10292                 if (port == PORT_A)
10293                         cpu_transcoder = TRANSCODER_DSI_A;
10294                 else
10295                         cpu_transcoder = TRANSCODER_DSI_C;
10296
10297                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10298                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10299
10300                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10301                 if (!wf)
10302                         continue;
10303
10304                 wakerefs[power_domain] = wf;
10305                 *power_domain_mask |= BIT_ULL(power_domain);
10306
10307                 /*
10308                  * The PLL needs to be enabled with a valid divider
10309                  * configuration, otherwise accessing DSI registers will hang
10310                  * the machine. See BSpec North Display Engine
10311                  * registers/MIPI[BXT]. We can break out here early, since we
10312                  * need the same DSI PLL to be enabled for both DSI ports.
10313                  */
10314                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10315                         break;
10316
10317                 /* XXX: this works for video mode only */
10318                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10319                 if (!(tmp & DPI_ENABLE))
10320                         continue;
10321
10322                 tmp = I915_READ(MIPI_CTRL(port));
10323                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10324                         continue;
10325
10326                 pipe_config->cpu_transcoder = cpu_transcoder;
10327                 break;
10328         }
10329
10330         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10331 }
10332
10333 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10334                                        struct intel_crtc_state *pipe_config)
10335 {
10336         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10337         struct intel_shared_dpll *pll;
10338         enum port port;
10339         u32 tmp;
10340
10341         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
10342
10343         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
10344
10345         if (INTEL_GEN(dev_priv) >= 11)
10346                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10347         else if (IS_CANNONLAKE(dev_priv))
10348                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10349         else if (IS_GEN9_BC(dev_priv))
10350                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10351         else if (IS_GEN9_LP(dev_priv))
10352                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10353         else
10354                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10355
10356         pll = pipe_config->shared_dpll;
10357         if (pll) {
10358                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10359                                                 &pipe_config->dpll_hw_state));
10360         }
10361
10362         /*
10363          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10364          * DDI E. So just check whether this pipe is wired to DDI E and whether
10365          * the PCH transcoder is on.
10366          */
10367         if (INTEL_GEN(dev_priv) < 9 &&
10368             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10369                 pipe_config->has_pch_encoder = true;
10370
10371                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10372                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10373                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10374
10375                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10376         }
10377 }
10378
10379 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10380                                     struct intel_crtc_state *pipe_config)
10381 {
10382         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10383         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10384         enum intel_display_power_domain power_domain;
10385         u64 power_domain_mask;
10386         bool active;
10387
10388         intel_crtc_init_scalers(crtc, pipe_config);
10389
10390         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10391         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10392         if (!wf)
10393                 return false;
10394
10395         wakerefs[power_domain] = wf;
10396         power_domain_mask = BIT_ULL(power_domain);
10397
10398         pipe_config->shared_dpll = NULL;
10399
10400         active = hsw_get_transcoder_state(crtc, pipe_config,
10401                                           &power_domain_mask, wakerefs);
10402
10403         if (IS_GEN9_LP(dev_priv) &&
10404             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10405                                          &power_domain_mask, wakerefs)) {
10406                 WARN_ON(active);
10407                 active = true;
10408         }
10409
10410         if (!active)
10411                 goto out;
10412
10413         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10414             INTEL_GEN(dev_priv) >= 11) {
10415                 haswell_get_ddi_port_state(crtc, pipe_config);
10416                 intel_get_pipe_timings(crtc, pipe_config);
10417         }
10418
10419         intel_get_pipe_src_size(crtc, pipe_config);
10420         intel_get_crtc_ycbcr_config(crtc, pipe_config);
10421
10422         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10423
10424         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10425
10426         if (INTEL_GEN(dev_priv) >= 9) {
10427                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10428
10429                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10430                         pipe_config->gamma_enable = true;
10431
10432                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10433                         pipe_config->csc_enable = true;
10434         } else {
10435                 i9xx_get_pipe_color_config(pipe_config);
10436         }
10437
10438         intel_color_get_config(pipe_config);
10439
10440         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10441         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10442
10443         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10444         if (wf) {
10445                 wakerefs[power_domain] = wf;
10446                 power_domain_mask |= BIT_ULL(power_domain);
10447
10448                 if (INTEL_GEN(dev_priv) >= 9)
10449                         skylake_get_pfit_config(crtc, pipe_config);
10450                 else
10451                         ironlake_get_pfit_config(crtc, pipe_config);
10452         }
10453
10454         if (hsw_crtc_supports_ips(crtc)) {
10455                 if (IS_HASWELL(dev_priv))
10456                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10457                 else {
10458                         /*
10459                          * We cannot readout IPS state on broadwell, set to
10460                          * true so we can set it to a defined state on first
10461                          * commit.
10462                          */
10463                         pipe_config->ips_enabled = true;
10464                 }
10465         }
10466
10467         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10468             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10469                 pipe_config->pixel_multiplier =
10470                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10471         } else {
10472                 pipe_config->pixel_multiplier = 1;
10473         }
10474
10475 out:
10476         for_each_power_domain(power_domain, power_domain_mask)
10477                 intel_display_power_put(dev_priv,
10478                                         power_domain, wakerefs[power_domain]);
10479
10480         return active;
10481 }
10482
10483 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10484 {
10485         struct drm_i915_private *dev_priv =
10486                 to_i915(plane_state->base.plane->dev);
10487         const struct drm_framebuffer *fb = plane_state->base.fb;
10488         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10489         u32 base;
10490
10491         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10492                 base = obj->phys_handle->busaddr;
10493         else
10494                 base = intel_plane_ggtt_offset(plane_state);
10495
10496         base += plane_state->color_plane[0].offset;
10497
10498         /* ILK+ do this automagically */
10499         if (HAS_GMCH(dev_priv) &&
10500             plane_state->base.rotation & DRM_MODE_ROTATE_180)
10501                 base += (plane_state->base.crtc_h *
10502                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
10503
10504         return base;
10505 }
10506
10507 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10508 {
10509         int x = plane_state->base.crtc_x;
10510         int y = plane_state->base.crtc_y;
10511         u32 pos = 0;
10512
10513         if (x < 0) {
10514                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10515                 x = -x;
10516         }
10517         pos |= x << CURSOR_X_SHIFT;
10518
10519         if (y < 0) {
10520                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10521                 y = -y;
10522         }
10523         pos |= y << CURSOR_Y_SHIFT;
10524
10525         return pos;
10526 }
10527
10528 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10529 {
10530         const struct drm_mode_config *config =
10531                 &plane_state->base.plane->dev->mode_config;
10532         int width = plane_state->base.crtc_w;
10533         int height = plane_state->base.crtc_h;
10534
10535         return width > 0 && width <= config->cursor_width &&
10536                 height > 0 && height <= config->cursor_height;
10537 }
10538
10539 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10540 {
10541         int src_x, src_y;
10542         u32 offset;
10543         int ret;
10544
10545         ret = intel_plane_compute_gtt(plane_state);
10546         if (ret)
10547                 return ret;
10548
10549         if (!plane_state->base.visible)
10550                 return 0;
10551
10552         src_x = plane_state->base.src_x >> 16;
10553         src_y = plane_state->base.src_y >> 16;
10554
10555         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
10556         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
10557                                                     plane_state, 0);
10558
10559         if (src_x != 0 || src_y != 0) {
10560                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
10561                 return -EINVAL;
10562         }
10563
10564         plane_state->color_plane[0].offset = offset;
10565
10566         return 0;
10567 }
10568
10569 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
10570                               struct intel_plane_state *plane_state)
10571 {
10572         const struct drm_framebuffer *fb = plane_state->base.fb;
10573         int ret;
10574
10575         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
10576                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
10577                 return -EINVAL;
10578         }
10579
10580         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
10581                                                   &crtc_state->base,
10582                                                   DRM_PLANE_HELPER_NO_SCALING,
10583                                                   DRM_PLANE_HELPER_NO_SCALING,
10584                                                   true, true);
10585         if (ret)
10586                 return ret;
10587
10588         ret = intel_cursor_check_surface(plane_state);
10589         if (ret)
10590                 return ret;
10591
10592         if (!plane_state->base.visible)
10593                 return 0;
10594
10595         ret = intel_plane_check_src_coordinates(plane_state);
10596         if (ret)
10597                 return ret;
10598
10599         return 0;
10600 }
10601
10602 static unsigned int
10603 i845_cursor_max_stride(struct intel_plane *plane,
10604                        u32 pixel_format, u64 modifier,
10605                        unsigned int rotation)
10606 {
10607         return 2048;
10608 }
10609
10610 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10611 {
10612         u32 cntl = 0;
10613
10614         if (crtc_state->gamma_enable)
10615                 cntl |= CURSOR_GAMMA_ENABLE;
10616
10617         return cntl;
10618 }
10619
10620 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
10621                            const struct intel_plane_state *plane_state)
10622 {
10623         return CURSOR_ENABLE |
10624                 CURSOR_FORMAT_ARGB |
10625                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
10626 }
10627
10628 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
10629 {
10630         int width = plane_state->base.crtc_w;
10631
10632         /*
10633          * 845g/865g are only limited by the width of their cursors,
10634          * the height is arbitrary up to the precision of the register.
10635          */
10636         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
10637 }
10638
10639 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
10640                              struct intel_plane_state *plane_state)
10641 {
10642         const struct drm_framebuffer *fb = plane_state->base.fb;
10643         int ret;
10644
10645         ret = intel_check_cursor(crtc_state, plane_state);
10646         if (ret)
10647                 return ret;
10648
10649         /* if we want to turn off the cursor ignore width and height */
10650         if (!fb)
10651                 return 0;
10652
10653         /* Check for which cursor types we support */
10654         if (!i845_cursor_size_ok(plane_state)) {
10655                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10656                           plane_state->base.crtc_w,
10657                           plane_state->base.crtc_h);
10658                 return -EINVAL;
10659         }
10660
10661         WARN_ON(plane_state->base.visible &&
10662                 plane_state->color_plane[0].stride != fb->pitches[0]);
10663
10664         switch (fb->pitches[0]) {
10665         case 256:
10666         case 512:
10667         case 1024:
10668         case 2048:
10669                 break;
10670         default:
10671                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
10672                               fb->pitches[0]);
10673                 return -EINVAL;
10674         }
10675
10676         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
10677
10678         return 0;
10679 }
10680
10681 static void i845_update_cursor(struct intel_plane *plane,
10682                                const struct intel_crtc_state *crtc_state,
10683                                const struct intel_plane_state *plane_state)
10684 {
10685         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10686         u32 cntl = 0, base = 0, pos = 0, size = 0;
10687         unsigned long irqflags;
10688
10689         if (plane_state && plane_state->base.visible) {
10690                 unsigned int width = plane_state->base.crtc_w;
10691                 unsigned int height = plane_state->base.crtc_h;
10692
10693                 cntl = plane_state->ctl |
10694                         i845_cursor_ctl_crtc(crtc_state);
10695
10696                 size = (height << 12) | width;
10697
10698                 base = intel_cursor_base(plane_state);
10699                 pos = intel_cursor_position(plane_state);
10700         }
10701
10702         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10703
10704         /* On these chipsets we can only modify the base/size/stride
10705          * whilst the cursor is disabled.
10706          */
10707         if (plane->cursor.base != base ||
10708             plane->cursor.size != size ||
10709             plane->cursor.cntl != cntl) {
10710                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
10711                 I915_WRITE_FW(CURBASE(PIPE_A), base);
10712                 I915_WRITE_FW(CURSIZE, size);
10713                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10714                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
10715
10716                 plane->cursor.base = base;
10717                 plane->cursor.size = size;
10718                 plane->cursor.cntl = cntl;
10719         } else {
10720                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10721         }
10722
10723         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10724 }
10725
10726 static void i845_disable_cursor(struct intel_plane *plane,
10727                                 const struct intel_crtc_state *crtc_state)
10728 {
10729         i845_update_cursor(plane, crtc_state, NULL);
10730 }
10731
10732 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
10733                                      enum pipe *pipe)
10734 {
10735         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10736         enum intel_display_power_domain power_domain;
10737         intel_wakeref_t wakeref;
10738         bool ret;
10739
10740         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
10741         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10742         if (!wakeref)
10743                 return false;
10744
10745         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
10746
10747         *pipe = PIPE_A;
10748
10749         intel_display_power_put(dev_priv, power_domain, wakeref);
10750
10751         return ret;
10752 }
10753
10754 static unsigned int
10755 i9xx_cursor_max_stride(struct intel_plane *plane,
10756                        u32 pixel_format, u64 modifier,
10757                        unsigned int rotation)
10758 {
10759         return plane->base.dev->mode_config.cursor_width * 4;
10760 }
10761
10762 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10763 {
10764         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10765         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10766         u32 cntl = 0;
10767
10768         if (INTEL_GEN(dev_priv) >= 11)
10769                 return cntl;
10770
10771         if (crtc_state->gamma_enable)
10772                 cntl = MCURSOR_GAMMA_ENABLE;
10773
10774         if (crtc_state->csc_enable)
10775                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
10776
10777         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10778                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10779
10780         return cntl;
10781 }
10782
10783 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
10784                            const struct intel_plane_state *plane_state)
10785 {
10786         struct drm_i915_private *dev_priv =
10787                 to_i915(plane_state->base.plane->dev);
10788         u32 cntl = 0;
10789
10790         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
10791                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10792
10793         switch (plane_state->base.crtc_w) {
10794         case 64:
10795                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10796                 break;
10797         case 128:
10798                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10799                 break;
10800         case 256:
10801                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10802                 break;
10803         default:
10804                 MISSING_CASE(plane_state->base.crtc_w);
10805                 return 0;
10806         }
10807
10808         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10809                 cntl |= MCURSOR_ROTATE_180;
10810
10811         return cntl;
10812 }
10813
10814 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10815 {
10816         struct drm_i915_private *dev_priv =
10817                 to_i915(plane_state->base.plane->dev);
10818         int width = plane_state->base.crtc_w;
10819         int height = plane_state->base.crtc_h;
10820
10821         if (!intel_cursor_size_ok(plane_state))
10822                 return false;
10823
10824         /* Cursor width is limited to a few power-of-two sizes */
10825         switch (width) {
10826         case 256:
10827         case 128:
10828         case 64:
10829                 break;
10830         default:
10831                 return false;
10832         }
10833
10834         /*
10835          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10836          * height from 8 lines up to the cursor width, when the
10837          * cursor is not rotated. Everything else requires square
10838          * cursors.
10839          */
10840         if (HAS_CUR_FBC(dev_priv) &&
10841             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10842                 if (height < 8 || height > width)
10843                         return false;
10844         } else {
10845                 if (height != width)
10846                         return false;
10847         }
10848
10849         return true;
10850 }
10851
10852 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10853                              struct intel_plane_state *plane_state)
10854 {
10855         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10856         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10857         const struct drm_framebuffer *fb = plane_state->base.fb;
10858         enum pipe pipe = plane->pipe;
10859         int ret;
10860
10861         ret = intel_check_cursor(crtc_state, plane_state);
10862         if (ret)
10863                 return ret;
10864
10865         /* if we want to turn off the cursor ignore width and height */
10866         if (!fb)
10867                 return 0;
10868
10869         /* Check for which cursor types we support */
10870         if (!i9xx_cursor_size_ok(plane_state)) {
10871                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10872                           plane_state->base.crtc_w,
10873                           plane_state->base.crtc_h);
10874                 return -EINVAL;
10875         }
10876
10877         WARN_ON(plane_state->base.visible &&
10878                 plane_state->color_plane[0].stride != fb->pitches[0]);
10879
10880         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10881                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10882                               fb->pitches[0], plane_state->base.crtc_w);
10883                 return -EINVAL;
10884         }
10885
10886         /*
10887          * There's something wrong with the cursor on CHV pipe C.
10888          * If it straddles the left edge of the screen then
10889          * moving it away from the edge or disabling it often
10890          * results in a pipe underrun, and often that can lead to
10891          * dead pipe (constant underrun reported, and it scans
10892          * out just a solid color). To recover from that, the
10893          * display power well must be turned off and on again.
10894          * Refuse the put the cursor into that compromised position.
10895          */
10896         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10897             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10898                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10899                 return -EINVAL;
10900         }
10901
10902         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10903
10904         return 0;
10905 }
10906
10907 static void i9xx_update_cursor(struct intel_plane *plane,
10908                                const struct intel_crtc_state *crtc_state,
10909                                const struct intel_plane_state *plane_state)
10910 {
10911         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10912         enum pipe pipe = plane->pipe;
10913         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10914         unsigned long irqflags;
10915
10916         if (plane_state && plane_state->base.visible) {
10917                 cntl = plane_state->ctl |
10918                         i9xx_cursor_ctl_crtc(crtc_state);
10919
10920                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10921                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10922
10923                 base = intel_cursor_base(plane_state);
10924                 pos = intel_cursor_position(plane_state);
10925         }
10926
10927         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10928
10929         /*
10930          * On some platforms writing CURCNTR first will also
10931          * cause CURPOS to be armed by the CURBASE write.
10932          * Without the CURCNTR write the CURPOS write would
10933          * arm itself. Thus we always update CURCNTR before
10934          * CURPOS.
10935          *
10936          * On other platforms CURPOS always requires the
10937          * CURBASE write to arm the update. Additonally
10938          * a write to any of the cursor register will cancel
10939          * an already armed cursor update. Thus leaving out
10940          * the CURBASE write after CURPOS could lead to a
10941          * cursor that doesn't appear to move, or even change
10942          * shape. Thus we always write CURBASE.
10943          *
10944          * The other registers are armed by by the CURBASE write
10945          * except when the plane is getting enabled at which time
10946          * the CURCNTR write arms the update.
10947          */
10948
10949         if (INTEL_GEN(dev_priv) >= 9)
10950                 skl_write_cursor_wm(plane, crtc_state);
10951
10952         if (plane->cursor.base != base ||
10953             plane->cursor.size != fbc_ctl ||
10954             plane->cursor.cntl != cntl) {
10955                 if (HAS_CUR_FBC(dev_priv))
10956                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10957                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10958                 I915_WRITE_FW(CURPOS(pipe), pos);
10959                 I915_WRITE_FW(CURBASE(pipe), base);
10960
10961                 plane->cursor.base = base;
10962                 plane->cursor.size = fbc_ctl;
10963                 plane->cursor.cntl = cntl;
10964         } else {
10965                 I915_WRITE_FW(CURPOS(pipe), pos);
10966                 I915_WRITE_FW(CURBASE(pipe), base);
10967         }
10968
10969         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10970 }
10971
10972 static void i9xx_disable_cursor(struct intel_plane *plane,
10973                                 const struct intel_crtc_state *crtc_state)
10974 {
10975         i9xx_update_cursor(plane, crtc_state, NULL);
10976 }
10977
10978 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10979                                      enum pipe *pipe)
10980 {
10981         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10982         enum intel_display_power_domain power_domain;
10983         intel_wakeref_t wakeref;
10984         bool ret;
10985         u32 val;
10986
10987         /*
10988          * Not 100% correct for planes that can move between pipes,
10989          * but that's only the case for gen2-3 which don't have any
10990          * display power wells.
10991          */
10992         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10993         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10994         if (!wakeref)
10995                 return false;
10996
10997         val = I915_READ(CURCNTR(plane->pipe));
10998
10999         ret = val & MCURSOR_MODE;
11000
11001         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11002                 *pipe = plane->pipe;
11003         else
11004                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11005                         MCURSOR_PIPE_SELECT_SHIFT;
11006
11007         intel_display_power_put(dev_priv, power_domain, wakeref);
11008
11009         return ret;
11010 }
11011
11012 /* VESA 640x480x72Hz mode to set on the pipe */
11013 static const struct drm_display_mode load_detect_mode = {
11014         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11015                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11016 };
11017
11018 struct drm_framebuffer *
11019 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11020                          struct drm_mode_fb_cmd2 *mode_cmd)
11021 {
11022         struct intel_framebuffer *intel_fb;
11023         int ret;
11024
11025         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11026         if (!intel_fb)
11027                 return ERR_PTR(-ENOMEM);
11028
11029         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11030         if (ret)
11031                 goto err;
11032
11033         return &intel_fb->base;
11034
11035 err:
11036         kfree(intel_fb);
11037         return ERR_PTR(ret);
11038 }
11039
11040 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11041                                         struct drm_crtc *crtc)
11042 {
11043         struct drm_plane *plane;
11044         struct drm_plane_state *plane_state;
11045         int ret, i;
11046
11047         ret = drm_atomic_add_affected_planes(state, crtc);
11048         if (ret)
11049                 return ret;
11050
11051         for_each_new_plane_in_state(state, plane, plane_state, i) {
11052                 if (plane_state->crtc != crtc)
11053                         continue;
11054
11055                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11056                 if (ret)
11057                         return ret;
11058
11059                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11060         }
11061
11062         return 0;
11063 }
11064
11065 int intel_get_load_detect_pipe(struct drm_connector *connector,
11066                                const struct drm_display_mode *mode,
11067                                struct intel_load_detect_pipe *old,
11068                                struct drm_modeset_acquire_ctx *ctx)
11069 {
11070         struct intel_crtc *intel_crtc;
11071         struct intel_encoder *intel_encoder =
11072                 intel_attached_encoder(connector);
11073         struct drm_crtc *possible_crtc;
11074         struct drm_encoder *encoder = &intel_encoder->base;
11075         struct drm_crtc *crtc = NULL;
11076         struct drm_device *dev = encoder->dev;
11077         struct drm_i915_private *dev_priv = to_i915(dev);
11078         struct drm_mode_config *config = &dev->mode_config;
11079         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11080         struct drm_connector_state *connector_state;
11081         struct intel_crtc_state *crtc_state;
11082         int ret, i = -1;
11083
11084         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11085                       connector->base.id, connector->name,
11086                       encoder->base.id, encoder->name);
11087
11088         old->restore_state = NULL;
11089
11090         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11091
11092         /*
11093          * Algorithm gets a little messy:
11094          *
11095          *   - if the connector already has an assigned crtc, use it (but make
11096          *     sure it's on first)
11097          *
11098          *   - try to find the first unused crtc that can drive this connector,
11099          *     and use that if we find one
11100          */
11101
11102         /* See if we already have a CRTC for this connector */
11103         if (connector->state->crtc) {
11104                 crtc = connector->state->crtc;
11105
11106                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11107                 if (ret)
11108                         goto fail;
11109
11110                 /* Make sure the crtc and connector are running */
11111                 goto found;
11112         }
11113
11114         /* Find an unused one (if possible) */
11115         for_each_crtc(dev, possible_crtc) {
11116                 i++;
11117                 if (!(encoder->possible_crtcs & (1 << i)))
11118                         continue;
11119
11120                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11121                 if (ret)
11122                         goto fail;
11123
11124                 if (possible_crtc->state->enable) {
11125                         drm_modeset_unlock(&possible_crtc->mutex);
11126                         continue;
11127                 }
11128
11129                 crtc = possible_crtc;
11130                 break;
11131         }
11132
11133         /*
11134          * If we didn't find an unused CRTC, don't use any.
11135          */
11136         if (!crtc) {
11137                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11138                 ret = -ENODEV;
11139                 goto fail;
11140         }
11141
11142 found:
11143         intel_crtc = to_intel_crtc(crtc);
11144
11145         state = drm_atomic_state_alloc(dev);
11146         restore_state = drm_atomic_state_alloc(dev);
11147         if (!state || !restore_state) {
11148                 ret = -ENOMEM;
11149                 goto fail;
11150         }
11151
11152         state->acquire_ctx = ctx;
11153         restore_state->acquire_ctx = ctx;
11154
11155         connector_state = drm_atomic_get_connector_state(state, connector);
11156         if (IS_ERR(connector_state)) {
11157                 ret = PTR_ERR(connector_state);
11158                 goto fail;
11159         }
11160
11161         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11162         if (ret)
11163                 goto fail;
11164
11165         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11166         if (IS_ERR(crtc_state)) {
11167                 ret = PTR_ERR(crtc_state);
11168                 goto fail;
11169         }
11170
11171         crtc_state->base.active = crtc_state->base.enable = true;
11172
11173         if (!mode)
11174                 mode = &load_detect_mode;
11175
11176         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
11177         if (ret)
11178                 goto fail;
11179
11180         ret = intel_modeset_disable_planes(state, crtc);
11181         if (ret)
11182                 goto fail;
11183
11184         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11185         if (!ret)
11186                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11187         if (!ret)
11188                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11189         if (ret) {
11190                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11191                 goto fail;
11192         }
11193
11194         ret = drm_atomic_commit(state);
11195         if (ret) {
11196                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11197                 goto fail;
11198         }
11199
11200         old->restore_state = restore_state;
11201         drm_atomic_state_put(state);
11202
11203         /* let the connector get through one full cycle before testing */
11204         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11205         return true;
11206
11207 fail:
11208         if (state) {
11209                 drm_atomic_state_put(state);
11210                 state = NULL;
11211         }
11212         if (restore_state) {
11213                 drm_atomic_state_put(restore_state);
11214                 restore_state = NULL;
11215         }
11216
11217         if (ret == -EDEADLK)
11218                 return ret;
11219
11220         return false;
11221 }
11222
11223 void intel_release_load_detect_pipe(struct drm_connector *connector,
11224                                     struct intel_load_detect_pipe *old,
11225                                     struct drm_modeset_acquire_ctx *ctx)
11226 {
11227         struct intel_encoder *intel_encoder =
11228                 intel_attached_encoder(connector);
11229         struct drm_encoder *encoder = &intel_encoder->base;
11230         struct drm_atomic_state *state = old->restore_state;
11231         int ret;
11232
11233         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11234                       connector->base.id, connector->name,
11235                       encoder->base.id, encoder->name);
11236
11237         if (!state)
11238                 return;
11239
11240         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11241         if (ret)
11242                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11243         drm_atomic_state_put(state);
11244 }
11245
11246 static int i9xx_pll_refclk(struct drm_device *dev,
11247                            const struct intel_crtc_state *pipe_config)
11248 {
11249         struct drm_i915_private *dev_priv = to_i915(dev);
11250         u32 dpll = pipe_config->dpll_hw_state.dpll;
11251
11252         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11253                 return dev_priv->vbt.lvds_ssc_freq;
11254         else if (HAS_PCH_SPLIT(dev_priv))
11255                 return 120000;
11256         else if (!IS_GEN(dev_priv, 2))
11257                 return 96000;
11258         else
11259                 return 48000;
11260 }
11261
11262 /* Returns the clock of the currently programmed mode of the given pipe. */
11263 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11264                                 struct intel_crtc_state *pipe_config)
11265 {
11266         struct drm_device *dev = crtc->base.dev;
11267         struct drm_i915_private *dev_priv = to_i915(dev);
11268         int pipe = pipe_config->cpu_transcoder;
11269         u32 dpll = pipe_config->dpll_hw_state.dpll;
11270         u32 fp;
11271         struct dpll clock;
11272         int port_clock;
11273         int refclk = i9xx_pll_refclk(dev, pipe_config);
11274
11275         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11276                 fp = pipe_config->dpll_hw_state.fp0;
11277         else
11278                 fp = pipe_config->dpll_hw_state.fp1;
11279
11280         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11281         if (IS_PINEVIEW(dev_priv)) {
11282                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11283                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11284         } else {
11285                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11286                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11287         }
11288
11289         if (!IS_GEN(dev_priv, 2)) {
11290                 if (IS_PINEVIEW(dev_priv))
11291                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11292                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11293                 else
11294                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11295                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11296
11297                 switch (dpll & DPLL_MODE_MASK) {
11298                 case DPLLB_MODE_DAC_SERIAL:
11299                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11300                                 5 : 10;
11301                         break;
11302                 case DPLLB_MODE_LVDS:
11303                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11304                                 7 : 14;
11305                         break;
11306                 default:
11307                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11308                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11309                         return;
11310                 }
11311
11312                 if (IS_PINEVIEW(dev_priv))
11313                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11314                 else
11315                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11316         } else {
11317                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11318                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11319
11320                 if (is_lvds) {
11321                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11322                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11323
11324                         if (lvds & LVDS_CLKB_POWER_UP)
11325                                 clock.p2 = 7;
11326                         else
11327                                 clock.p2 = 14;
11328                 } else {
11329                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11330                                 clock.p1 = 2;
11331                         else {
11332                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11333                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11334                         }
11335                         if (dpll & PLL_P2_DIVIDE_BY_4)
11336                                 clock.p2 = 4;
11337                         else
11338                                 clock.p2 = 2;
11339                 }
11340
11341                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11342         }
11343
11344         /*
11345          * This value includes pixel_multiplier. We will use
11346          * port_clock to compute adjusted_mode.crtc_clock in the
11347          * encoder's get_config() function.
11348          */
11349         pipe_config->port_clock = port_clock;
11350 }
11351
11352 int intel_dotclock_calculate(int link_freq,
11353                              const struct intel_link_m_n *m_n)
11354 {
11355         /*
11356          * The calculation for the data clock is:
11357          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11358          * But we want to avoid losing precison if possible, so:
11359          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11360          *
11361          * and the link clock is simpler:
11362          * link_clock = (m * link_clock) / n
11363          */
11364
11365         if (!m_n->link_n)
11366                 return 0;
11367
11368         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11369 }
11370
11371 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11372                                    struct intel_crtc_state *pipe_config)
11373 {
11374         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11375
11376         /* read out port_clock from the DPLL */
11377         i9xx_crtc_clock_get(crtc, pipe_config);
11378
11379         /*
11380          * In case there is an active pipe without active ports,
11381          * we may need some idea for the dotclock anyway.
11382          * Calculate one based on the FDI configuration.
11383          */
11384         pipe_config->base.adjusted_mode.crtc_clock =
11385                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11386                                          &pipe_config->fdi_m_n);
11387 }
11388
11389 /* Returns the currently programmed mode of the given encoder. */
11390 struct drm_display_mode *
11391 intel_encoder_current_mode(struct intel_encoder *encoder)
11392 {
11393         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11394         struct intel_crtc_state *crtc_state;
11395         struct drm_display_mode *mode;
11396         struct intel_crtc *crtc;
11397         enum pipe pipe;
11398
11399         if (!encoder->get_hw_state(encoder, &pipe))
11400                 return NULL;
11401
11402         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11403
11404         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11405         if (!mode)
11406                 return NULL;
11407
11408         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
11409         if (!crtc_state) {
11410                 kfree(mode);
11411                 return NULL;
11412         }
11413
11414         crtc_state->base.crtc = &crtc->base;
11415
11416         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11417                 kfree(crtc_state);
11418                 kfree(mode);
11419                 return NULL;
11420         }
11421
11422         encoder->get_config(encoder, crtc_state);
11423
11424         intel_mode_from_pipe_config(mode, crtc_state);
11425
11426         kfree(crtc_state);
11427
11428         return mode;
11429 }
11430
11431 static void intel_crtc_destroy(struct drm_crtc *crtc)
11432 {
11433         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11434
11435         drm_crtc_cleanup(crtc);
11436         kfree(intel_crtc);
11437 }
11438
11439 /**
11440  * intel_wm_need_update - Check whether watermarks need updating
11441  * @cur: current plane state
11442  * @new: new plane state
11443  *
11444  * Check current plane state versus the new one to determine whether
11445  * watermarks need to be recalculated.
11446  *
11447  * Returns true or false.
11448  */
11449 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11450                                  struct intel_plane_state *new)
11451 {
11452         /* Update watermarks on tiling or size changes. */
11453         if (new->base.visible != cur->base.visible)
11454                 return true;
11455
11456         if (!cur->base.fb || !new->base.fb)
11457                 return false;
11458
11459         if (cur->base.fb->modifier != new->base.fb->modifier ||
11460             cur->base.rotation != new->base.rotation ||
11461             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
11462             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
11463             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
11464             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
11465                 return true;
11466
11467         return false;
11468 }
11469
11470 static bool needs_scaling(const struct intel_plane_state *state)
11471 {
11472         int src_w = drm_rect_width(&state->base.src) >> 16;
11473         int src_h = drm_rect_height(&state->base.src) >> 16;
11474         int dst_w = drm_rect_width(&state->base.dst);
11475         int dst_h = drm_rect_height(&state->base.dst);
11476
11477         return (src_w != dst_w || src_h != dst_h);
11478 }
11479
11480 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11481                                     struct intel_crtc_state *crtc_state,
11482                                     const struct intel_plane_state *old_plane_state,
11483                                     struct intel_plane_state *plane_state)
11484 {
11485         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11486         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
11487         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11488         bool mode_changed = needs_modeset(crtc_state);
11489         bool was_crtc_enabled = old_crtc_state->base.active;
11490         bool is_crtc_enabled = crtc_state->base.active;
11491         bool turn_off, turn_on, visible, was_visible;
11492         struct drm_framebuffer *fb = plane_state->base.fb;
11493         int ret;
11494
11495         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
11496                 ret = skl_update_scaler_plane(crtc_state, plane_state);
11497                 if (ret)
11498                         return ret;
11499         }
11500
11501         was_visible = old_plane_state->base.visible;
11502         visible = plane_state->base.visible;
11503
11504         if (!was_crtc_enabled && WARN_ON(was_visible))
11505                 was_visible = false;
11506
11507         /*
11508          * Visibility is calculated as if the crtc was on, but
11509          * after scaler setup everything depends on it being off
11510          * when the crtc isn't active.
11511          *
11512          * FIXME this is wrong for watermarks. Watermarks should also
11513          * be computed as if the pipe would be active. Perhaps move
11514          * per-plane wm computation to the .check_plane() hook, and
11515          * only combine the results from all planes in the current place?
11516          */
11517         if (!is_crtc_enabled) {
11518                 plane_state->base.visible = visible = false;
11519                 crtc_state->active_planes &= ~BIT(plane->id);
11520                 crtc_state->data_rate[plane->id] = 0;
11521         }
11522
11523         if (!was_visible && !visible)
11524                 return 0;
11525
11526         if (fb != old_plane_state->base.fb)
11527                 crtc_state->fb_changed = true;
11528
11529         turn_off = was_visible && (!visible || mode_changed);
11530         turn_on = visible && (!was_visible || mode_changed);
11531
11532         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
11533                          crtc->base.base.id, crtc->base.name,
11534                          plane->base.base.id, plane->base.name,
11535                          fb ? fb->base.id : -1);
11536
11537         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
11538                          plane->base.base.id, plane->base.name,
11539                          was_visible, visible,
11540                          turn_off, turn_on, mode_changed);
11541
11542         if (turn_on) {
11543                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11544                         crtc_state->update_wm_pre = true;
11545
11546                 /* must disable cxsr around plane enable/disable */
11547                 if (plane->id != PLANE_CURSOR)
11548                         crtc_state->disable_cxsr = true;
11549         } else if (turn_off) {
11550                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11551                         crtc_state->update_wm_post = true;
11552
11553                 /* must disable cxsr around plane enable/disable */
11554                 if (plane->id != PLANE_CURSOR)
11555                         crtc_state->disable_cxsr = true;
11556         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
11557                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
11558                         /* FIXME bollocks */
11559                         crtc_state->update_wm_pre = true;
11560                         crtc_state->update_wm_post = true;
11561                 }
11562         }
11563
11564         if (visible || was_visible)
11565                 crtc_state->fb_bits |= plane->frontbuffer_bit;
11566
11567         /*
11568          * ILK/SNB DVSACNTR/Sprite Enable
11569          * IVB SPR_CTL/Sprite Enable
11570          * "When in Self Refresh Big FIFO mode, a write to enable the
11571          *  plane will be internally buffered and delayed while Big FIFO
11572          *  mode is exiting."
11573          *
11574          * Which means that enabling the sprite can take an extra frame
11575          * when we start in big FIFO mode (LP1+). Thus we need to drop
11576          * down to LP0 and wait for vblank in order to make sure the
11577          * sprite gets enabled on the next vblank after the register write.
11578          * Doing otherwise would risk enabling the sprite one frame after
11579          * we've already signalled flip completion. We can resume LP1+
11580          * once the sprite has been enabled.
11581          *
11582          *
11583          * WaCxSRDisabledForSpriteScaling:ivb
11584          * IVB SPR_SCALE/Scaling Enable
11585          * "Low Power watermarks must be disabled for at least one
11586          *  frame before enabling sprite scaling, and kept disabled
11587          *  until sprite scaling is disabled."
11588          *
11589          * ILK/SNB DVSASCALE/Scaling Enable
11590          * "When in Self Refresh Big FIFO mode, scaling enable will be
11591          *  masked off while Big FIFO mode is exiting."
11592          *
11593          * Despite the w/a only being listed for IVB we assume that
11594          * the ILK/SNB note has similar ramifications, hence we apply
11595          * the w/a on all three platforms.
11596          *
11597          * With experimental results seems this is needed also for primary
11598          * plane, not only sprite plane.
11599          */
11600         if (plane->id != PLANE_CURSOR &&
11601             (IS_GEN_RANGE(dev_priv, 5, 6) ||
11602              IS_IVYBRIDGE(dev_priv)) &&
11603             (turn_on || (!needs_scaling(old_plane_state) &&
11604                          needs_scaling(plane_state))))
11605                 crtc_state->disable_lp_wm = true;
11606
11607         return 0;
11608 }
11609
11610 static bool encoders_cloneable(const struct intel_encoder *a,
11611                                const struct intel_encoder *b)
11612 {
11613         /* masks could be asymmetric, so check both ways */
11614         return a == b || (a->cloneable & (1 << b->type) &&
11615                           b->cloneable & (1 << a->type));
11616 }
11617
11618 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11619                                          struct intel_crtc *crtc,
11620                                          struct intel_encoder *encoder)
11621 {
11622         struct intel_encoder *source_encoder;
11623         struct drm_connector *connector;
11624         struct drm_connector_state *connector_state;
11625         int i;
11626
11627         for_each_new_connector_in_state(state, connector, connector_state, i) {
11628                 if (connector_state->crtc != &crtc->base)
11629                         continue;
11630
11631                 source_encoder =
11632                         to_intel_encoder(connector_state->best_encoder);
11633                 if (!encoders_cloneable(encoder, source_encoder))
11634                         return false;
11635         }
11636
11637         return true;
11638 }
11639
11640 static int icl_add_linked_planes(struct intel_atomic_state *state)
11641 {
11642         struct intel_plane *plane, *linked;
11643         struct intel_plane_state *plane_state, *linked_plane_state;
11644         int i;
11645
11646         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11647                 linked = plane_state->linked_plane;
11648
11649                 if (!linked)
11650                         continue;
11651
11652                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
11653                 if (IS_ERR(linked_plane_state))
11654                         return PTR_ERR(linked_plane_state);
11655
11656                 WARN_ON(linked_plane_state->linked_plane != plane);
11657                 WARN_ON(linked_plane_state->slave == plane_state->slave);
11658         }
11659
11660         return 0;
11661 }
11662
11663 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
11664 {
11665         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11666         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11667         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
11668         struct intel_plane *plane, *linked;
11669         struct intel_plane_state *plane_state;
11670         int i;
11671
11672         if (INTEL_GEN(dev_priv) < 11)
11673                 return 0;
11674
11675         /*
11676          * Destroy all old plane links and make the slave plane invisible
11677          * in the crtc_state->active_planes mask.
11678          */
11679         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11680                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
11681                         continue;
11682
11683                 plane_state->linked_plane = NULL;
11684                 if (plane_state->slave && !plane_state->base.visible) {
11685                         crtc_state->active_planes &= ~BIT(plane->id);
11686                         crtc_state->update_planes |= BIT(plane->id);
11687                 }
11688
11689                 plane_state->slave = false;
11690         }
11691
11692         if (!crtc_state->nv12_planes)
11693                 return 0;
11694
11695         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11696                 struct intel_plane_state *linked_state = NULL;
11697
11698                 if (plane->pipe != crtc->pipe ||
11699                     !(crtc_state->nv12_planes & BIT(plane->id)))
11700                         continue;
11701
11702                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
11703                         if (!icl_is_nv12_y_plane(linked->id))
11704                                 continue;
11705
11706                         if (crtc_state->active_planes & BIT(linked->id))
11707                                 continue;
11708
11709                         linked_state = intel_atomic_get_plane_state(state, linked);
11710                         if (IS_ERR(linked_state))
11711                                 return PTR_ERR(linked_state);
11712
11713                         break;
11714                 }
11715
11716                 if (!linked_state) {
11717                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
11718                                       hweight8(crtc_state->nv12_planes));
11719
11720                         return -EINVAL;
11721                 }
11722
11723                 plane_state->linked_plane = linked;
11724
11725                 linked_state->slave = true;
11726                 linked_state->linked_plane = plane;
11727                 crtc_state->active_planes |= BIT(linked->id);
11728                 crtc_state->update_planes |= BIT(linked->id);
11729                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
11730         }
11731
11732         return 0;
11733 }
11734
11735 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
11736 {
11737         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
11738         struct intel_atomic_state *state =
11739                 to_intel_atomic_state(new_crtc_state->base.state);
11740         const struct intel_crtc_state *old_crtc_state =
11741                 intel_atomic_get_old_crtc_state(state, crtc);
11742
11743         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
11744 }
11745
11746 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11747                                    struct drm_crtc_state *crtc_state)
11748 {
11749         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11750         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11751         struct intel_crtc_state *pipe_config =
11752                 to_intel_crtc_state(crtc_state);
11753         int ret;
11754         bool mode_changed = needs_modeset(pipe_config);
11755
11756         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
11757             mode_changed && !crtc_state->active)
11758                 pipe_config->update_wm_post = true;
11759
11760         if (mode_changed && crtc_state->enable &&
11761             dev_priv->display.crtc_compute_clock &&
11762             !WARN_ON(pipe_config->shared_dpll)) {
11763                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11764                                                            pipe_config);
11765                 if (ret)
11766                         return ret;
11767         }
11768
11769         /*
11770          * May need to update pipe gamma enable bits
11771          * when C8 planes are getting enabled/disabled.
11772          */
11773         if (c8_planes_changed(pipe_config))
11774                 crtc_state->color_mgmt_changed = true;
11775
11776         if (mode_changed || pipe_config->update_pipe ||
11777             crtc_state->color_mgmt_changed) {
11778                 ret = intel_color_check(pipe_config);
11779                 if (ret)
11780                         return ret;
11781         }
11782
11783         ret = 0;
11784         if (dev_priv->display.compute_pipe_wm) {
11785                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
11786                 if (ret) {
11787                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
11788                         return ret;
11789                 }
11790         }
11791
11792         if (dev_priv->display.compute_intermediate_wm) {
11793                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
11794                         return 0;
11795
11796                 /*
11797                  * Calculate 'intermediate' watermarks that satisfy both the
11798                  * old state and the new state.  We can program these
11799                  * immediately.
11800                  */
11801                 ret = dev_priv->display.compute_intermediate_wm(pipe_config);
11802                 if (ret) {
11803                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11804                         return ret;
11805                 }
11806         }
11807
11808         if (INTEL_GEN(dev_priv) >= 9) {
11809                 if (mode_changed || pipe_config->update_pipe)
11810                         ret = skl_update_scaler_crtc(pipe_config);
11811
11812                 if (!ret)
11813                         ret = icl_check_nv12_planes(pipe_config);
11814                 if (!ret)
11815                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11816                                                             pipe_config);
11817                 if (!ret)
11818                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11819                                                          pipe_config);
11820         }
11821
11822         if (HAS_IPS(dev_priv))
11823                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11824
11825         return ret;
11826 }
11827
11828 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11829         .atomic_check = intel_crtc_atomic_check,
11830 };
11831
11832 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11833 {
11834         struct intel_connector *connector;
11835         struct drm_connector_list_iter conn_iter;
11836
11837         drm_connector_list_iter_begin(dev, &conn_iter);
11838         for_each_intel_connector_iter(connector, &conn_iter) {
11839                 if (connector->base.state->crtc)
11840                         drm_connector_put(&connector->base);
11841
11842                 if (connector->base.encoder) {
11843                         connector->base.state->best_encoder =
11844                                 connector->base.encoder;
11845                         connector->base.state->crtc =
11846                                 connector->base.encoder->crtc;
11847
11848                         drm_connector_get(&connector->base);
11849                 } else {
11850                         connector->base.state->best_encoder = NULL;
11851                         connector->base.state->crtc = NULL;
11852                 }
11853         }
11854         drm_connector_list_iter_end(&conn_iter);
11855 }
11856
11857 static int
11858 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11859                       struct intel_crtc_state *pipe_config)
11860 {
11861         struct drm_connector *connector = conn_state->connector;
11862         const struct drm_display_info *info = &connector->display_info;
11863         int bpp;
11864
11865         switch (conn_state->max_bpc) {
11866         case 6 ... 7:
11867                 bpp = 6 * 3;
11868                 break;
11869         case 8 ... 9:
11870                 bpp = 8 * 3;
11871                 break;
11872         case 10 ... 11:
11873                 bpp = 10 * 3;
11874                 break;
11875         case 12:
11876                 bpp = 12 * 3;
11877                 break;
11878         default:
11879                 return -EINVAL;
11880         }
11881
11882         if (bpp < pipe_config->pipe_bpp) {
11883                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11884                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11885                               connector->base.id, connector->name,
11886                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11887                               pipe_config->pipe_bpp);
11888
11889                 pipe_config->pipe_bpp = bpp;
11890         }
11891
11892         return 0;
11893 }
11894
11895 static int
11896 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11897                           struct intel_crtc_state *pipe_config)
11898 {
11899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11900         struct drm_atomic_state *state = pipe_config->base.state;
11901         struct drm_connector *connector;
11902         struct drm_connector_state *connector_state;
11903         int bpp, i;
11904
11905         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11906             IS_CHERRYVIEW(dev_priv)))
11907                 bpp = 10*3;
11908         else if (INTEL_GEN(dev_priv) >= 5)
11909                 bpp = 12*3;
11910         else
11911                 bpp = 8*3;
11912
11913         pipe_config->pipe_bpp = bpp;
11914
11915         /* Clamp display bpp to connector max bpp */
11916         for_each_new_connector_in_state(state, connector, connector_state, i) {
11917                 int ret;
11918
11919                 if (connector_state->crtc != &crtc->base)
11920                         continue;
11921
11922                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11923                 if (ret)
11924                         return ret;
11925         }
11926
11927         return 0;
11928 }
11929
11930 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11931 {
11932         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11933                       "type: 0x%x flags: 0x%x\n",
11934                       mode->crtc_clock,
11935                       mode->crtc_hdisplay, mode->crtc_hsync_start,
11936                       mode->crtc_hsync_end, mode->crtc_htotal,
11937                       mode->crtc_vdisplay, mode->crtc_vsync_start,
11938                       mode->crtc_vsync_end, mode->crtc_vtotal,
11939                       mode->type, mode->flags);
11940 }
11941
11942 static inline void
11943 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
11944                       const char *id, unsigned int lane_count,
11945                       const struct intel_link_m_n *m_n)
11946 {
11947         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11948                       id, lane_count,
11949                       m_n->gmch_m, m_n->gmch_n,
11950                       m_n->link_m, m_n->link_n, m_n->tu);
11951 }
11952
11953 static void
11954 intel_dump_infoframe(struct drm_i915_private *dev_priv,
11955                      const union hdmi_infoframe *frame)
11956 {
11957         if ((drm_debug & DRM_UT_KMS) == 0)
11958                 return;
11959
11960         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
11961 }
11962
11963 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11964
11965 static const char * const output_type_str[] = {
11966         OUTPUT_TYPE(UNUSED),
11967         OUTPUT_TYPE(ANALOG),
11968         OUTPUT_TYPE(DVO),
11969         OUTPUT_TYPE(SDVO),
11970         OUTPUT_TYPE(LVDS),
11971         OUTPUT_TYPE(TVOUT),
11972         OUTPUT_TYPE(HDMI),
11973         OUTPUT_TYPE(DP),
11974         OUTPUT_TYPE(EDP),
11975         OUTPUT_TYPE(DSI),
11976         OUTPUT_TYPE(DDI),
11977         OUTPUT_TYPE(DP_MST),
11978 };
11979
11980 #undef OUTPUT_TYPE
11981
11982 static void snprintf_output_types(char *buf, size_t len,
11983                                   unsigned int output_types)
11984 {
11985         char *str = buf;
11986         int i;
11987
11988         str[0] = '\0';
11989
11990         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
11991                 int r;
11992
11993                 if ((output_types & BIT(i)) == 0)
11994                         continue;
11995
11996                 r = snprintf(str, len, "%s%s",
11997                              str != buf ? "," : "", output_type_str[i]);
11998                 if (r >= len)
11999                         break;
12000                 str += r;
12001                 len -= r;
12002
12003                 output_types &= ~BIT(i);
12004         }
12005
12006         WARN_ON_ONCE(output_types != 0);
12007 }
12008
12009 static const char * const output_format_str[] = {
12010         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12011         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12012         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12013         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12014 };
12015
12016 static const char *output_formats(enum intel_output_format format)
12017 {
12018         if (format >= ARRAY_SIZE(output_format_str))
12019                 format = INTEL_OUTPUT_FORMAT_INVALID;
12020         return output_format_str[format];
12021 }
12022
12023 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12024 {
12025         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
12026         const struct drm_framebuffer *fb = plane_state->base.fb;
12027         struct drm_format_name_buf format_name;
12028
12029         if (!fb) {
12030                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12031                               plane->base.base.id, plane->base.name,
12032                               yesno(plane_state->base.visible));
12033                 return;
12034         }
12035
12036         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12037                       plane->base.base.id, plane->base.name,
12038                       fb->base.id, fb->width, fb->height,
12039                       drm_get_format_name(fb->format->format, &format_name),
12040                       yesno(plane_state->base.visible));
12041         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12042                       plane_state->base.rotation, plane_state->scaler_id);
12043         if (plane_state->base.visible)
12044                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12045                               DRM_RECT_FP_ARG(&plane_state->base.src),
12046                               DRM_RECT_ARG(&plane_state->base.dst));
12047 }
12048
12049 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12050                                    struct intel_atomic_state *state,
12051                                    const char *context)
12052 {
12053         struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
12054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12055         const struct intel_plane_state *plane_state;
12056         struct intel_plane *plane;
12057         char buf[64];
12058         int i;
12059
12060         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12061                       crtc->base.base.id, crtc->base.name,
12062                       yesno(pipe_config->base.enable), context);
12063
12064         if (!pipe_config->base.enable)
12065                 goto dump_planes;
12066
12067         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12068         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12069                       yesno(pipe_config->base.active),
12070                       buf, pipe_config->output_types,
12071                       output_formats(pipe_config->output_format));
12072
12073         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12074                       transcoder_name(pipe_config->cpu_transcoder),
12075                       pipe_config->pipe_bpp, pipe_config->dither);
12076
12077         if (pipe_config->has_pch_encoder)
12078                 intel_dump_m_n_config(pipe_config, "fdi",
12079                                       pipe_config->fdi_lanes,
12080                                       &pipe_config->fdi_m_n);
12081
12082         if (intel_crtc_has_dp_encoder(pipe_config)) {
12083                 intel_dump_m_n_config(pipe_config, "dp m_n",
12084                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12085                 if (pipe_config->has_drrs)
12086                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12087                                               pipe_config->lane_count,
12088                                               &pipe_config->dp_m2_n2);
12089         }
12090
12091         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12092                       pipe_config->has_audio, pipe_config->has_infoframe,
12093                       pipe_config->infoframes.enable);
12094
12095         if (pipe_config->infoframes.enable &
12096             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12097                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12098         if (pipe_config->infoframes.enable &
12099             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12100                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12101         if (pipe_config->infoframes.enable &
12102             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12103                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12104         if (pipe_config->infoframes.enable &
12105             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12106                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12107
12108         DRM_DEBUG_KMS("requested mode:\n");
12109         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12110         DRM_DEBUG_KMS("adjusted mode:\n");
12111         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12112         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12113         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12114                       pipe_config->port_clock,
12115                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12116                       pipe_config->pixel_rate);
12117
12118         if (INTEL_GEN(dev_priv) >= 9)
12119                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12120                               crtc->num_scalers,
12121                               pipe_config->scaler_state.scaler_users,
12122                               pipe_config->scaler_state.scaler_id);
12123
12124         if (HAS_GMCH(dev_priv))
12125                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12126                               pipe_config->gmch_pfit.control,
12127                               pipe_config->gmch_pfit.pgm_ratios,
12128                               pipe_config->gmch_pfit.lvds_border_bits);
12129         else
12130                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12131                               pipe_config->pch_pfit.pos,
12132                               pipe_config->pch_pfit.size,
12133                               enableddisabled(pipe_config->pch_pfit.enabled),
12134                               yesno(pipe_config->pch_pfit.force_thru));
12135
12136         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12137                       pipe_config->ips_enabled, pipe_config->double_wide);
12138
12139         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12140
12141 dump_planes:
12142         if (!state)
12143                 return;
12144
12145         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12146                 if (plane->pipe == crtc->pipe)
12147                         intel_dump_plane_state(plane_state);
12148         }
12149 }
12150
12151 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12152 {
12153         struct drm_device *dev = state->base.dev;
12154         struct drm_connector *connector;
12155         struct drm_connector_list_iter conn_iter;
12156         unsigned int used_ports = 0;
12157         unsigned int used_mst_ports = 0;
12158         bool ret = true;
12159
12160         /*
12161          * Walk the connector list instead of the encoder
12162          * list to detect the problem on ddi platforms
12163          * where there's just one encoder per digital port.
12164          */
12165         drm_connector_list_iter_begin(dev, &conn_iter);
12166         drm_for_each_connector_iter(connector, &conn_iter) {
12167                 struct drm_connector_state *connector_state;
12168                 struct intel_encoder *encoder;
12169
12170                 connector_state =
12171                         drm_atomic_get_new_connector_state(&state->base,
12172                                                            connector);
12173                 if (!connector_state)
12174                         connector_state = connector->state;
12175
12176                 if (!connector_state->best_encoder)
12177                         continue;
12178
12179                 encoder = to_intel_encoder(connector_state->best_encoder);
12180
12181                 WARN_ON(!connector_state->crtc);
12182
12183                 switch (encoder->type) {
12184                         unsigned int port_mask;
12185                 case INTEL_OUTPUT_DDI:
12186                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12187                                 break;
12188                         /* else: fall through */
12189                 case INTEL_OUTPUT_DP:
12190                 case INTEL_OUTPUT_HDMI:
12191                 case INTEL_OUTPUT_EDP:
12192                         port_mask = 1 << encoder->port;
12193
12194                         /* the same port mustn't appear more than once */
12195                         if (used_ports & port_mask)
12196                                 ret = false;
12197
12198                         used_ports |= port_mask;
12199                         break;
12200                 case INTEL_OUTPUT_DP_MST:
12201                         used_mst_ports |=
12202                                 1 << encoder->port;
12203                         break;
12204                 default:
12205                         break;
12206                 }
12207         }
12208         drm_connector_list_iter_end(&conn_iter);
12209
12210         /* can't mix MST and SST/HDMI on the same port */
12211         if (used_ports & used_mst_ports)
12212                 return false;
12213
12214         return ret;
12215 }
12216
12217 static int
12218 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12219 {
12220         struct drm_i915_private *dev_priv =
12221                 to_i915(crtc_state->base.crtc->dev);
12222         struct intel_crtc_state *saved_state;
12223
12224         saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL);
12225         if (!saved_state)
12226                 return -ENOMEM;
12227
12228         /* FIXME: before the switch to atomic started, a new pipe_config was
12229          * kzalloc'd. Code that depends on any field being zero should be
12230          * fixed, so that the crtc_state can be safely duplicated. For now,
12231          * only fields that are know to not cause problems are preserved. */
12232
12233         saved_state->scaler_state = crtc_state->scaler_state;
12234         saved_state->shared_dpll = crtc_state->shared_dpll;
12235         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12236         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12237                sizeof(saved_state->icl_port_dplls));
12238         saved_state->crc_enabled = crtc_state->crc_enabled;
12239         if (IS_G4X(dev_priv) ||
12240             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12241                 saved_state->wm = crtc_state->wm;
12242
12243         /* Keep base drm_crtc_state intact, only clear our extended struct */
12244         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
12245         memcpy(&crtc_state->base + 1, &saved_state->base + 1,
12246                sizeof(*crtc_state) - sizeof(crtc_state->base));
12247
12248         kfree(saved_state);
12249         return 0;
12250 }
12251
12252 static int
12253 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12254 {
12255         struct drm_crtc *crtc = pipe_config->base.crtc;
12256         struct drm_atomic_state *state = pipe_config->base.state;
12257         struct intel_encoder *encoder;
12258         struct drm_connector *connector;
12259         struct drm_connector_state *connector_state;
12260         int base_bpp, ret;
12261         int i;
12262         bool retry = true;
12263
12264         ret = clear_intel_crtc_state(pipe_config);
12265         if (ret)
12266                 return ret;
12267
12268         pipe_config->cpu_transcoder =
12269                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12270
12271         /*
12272          * Sanitize sync polarity flags based on requested ones. If neither
12273          * positive or negative polarity is requested, treat this as meaning
12274          * negative polarity.
12275          */
12276         if (!(pipe_config->base.adjusted_mode.flags &
12277               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12278                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12279
12280         if (!(pipe_config->base.adjusted_mode.flags &
12281               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12282                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12283
12284         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12285                                         pipe_config);
12286         if (ret)
12287                 return ret;
12288
12289         base_bpp = pipe_config->pipe_bpp;
12290
12291         /*
12292          * Determine the real pipe dimensions. Note that stereo modes can
12293          * increase the actual pipe size due to the frame doubling and
12294          * insertion of additional space for blanks between the frame. This
12295          * is stored in the crtc timings. We use the requested mode to do this
12296          * computation to clearly distinguish it from the adjusted mode, which
12297          * can be changed by the connectors in the below retry loop.
12298          */
12299         drm_mode_get_hv_timing(&pipe_config->base.mode,
12300                                &pipe_config->pipe_src_w,
12301                                &pipe_config->pipe_src_h);
12302
12303         for_each_new_connector_in_state(state, connector, connector_state, i) {
12304                 if (connector_state->crtc != crtc)
12305                         continue;
12306
12307                 encoder = to_intel_encoder(connector_state->best_encoder);
12308
12309                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12310                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12311                         return -EINVAL;
12312                 }
12313
12314                 /*
12315                  * Determine output_types before calling the .compute_config()
12316                  * hooks so that the hooks can use this information safely.
12317                  */
12318                 if (encoder->compute_output_type)
12319                         pipe_config->output_types |=
12320                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12321                                                                  connector_state));
12322                 else
12323                         pipe_config->output_types |= BIT(encoder->type);
12324         }
12325
12326 encoder_retry:
12327         /* Ensure the port clock defaults are reset when retrying. */
12328         pipe_config->port_clock = 0;
12329         pipe_config->pixel_multiplier = 1;
12330
12331         /* Fill in default crtc timings, allow encoders to overwrite them. */
12332         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12333                               CRTC_STEREO_DOUBLE);
12334
12335         /* Pass our mode to the connectors and the CRTC to give them a chance to
12336          * adjust it according to limitations or connector properties, and also
12337          * a chance to reject the mode entirely.
12338          */
12339         for_each_new_connector_in_state(state, connector, connector_state, i) {
12340                 if (connector_state->crtc != crtc)
12341                         continue;
12342
12343                 encoder = to_intel_encoder(connector_state->best_encoder);
12344                 ret = encoder->compute_config(encoder, pipe_config,
12345                                               connector_state);
12346                 if (ret < 0) {
12347                         if (ret != -EDEADLK)
12348                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
12349                                               ret);
12350                         return ret;
12351                 }
12352         }
12353
12354         /* Set default port clock if not overwritten by the encoder. Needs to be
12355          * done afterwards in case the encoder adjusts the mode. */
12356         if (!pipe_config->port_clock)
12357                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12358                         * pipe_config->pixel_multiplier;
12359
12360         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12361         if (ret == -EDEADLK)
12362                 return ret;
12363         if (ret < 0) {
12364                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12365                 return ret;
12366         }
12367
12368         if (ret == RETRY) {
12369                 if (WARN(!retry, "loop in pipe configuration computation\n"))
12370                         return -EINVAL;
12371
12372                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12373                 retry = false;
12374                 goto encoder_retry;
12375         }
12376
12377         /* Dithering seems to not pass-through bits correctly when it should, so
12378          * only enable it on 6bpc panels and when its not a compliance
12379          * test requesting 6bpc video pattern.
12380          */
12381         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
12382                 !pipe_config->dither_force_disable;
12383         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12384                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12385
12386         return 0;
12387 }
12388
12389 bool intel_fuzzy_clock_check(int clock1, int clock2)
12390 {
12391         int diff;
12392
12393         if (clock1 == clock2)
12394                 return true;
12395
12396         if (!clock1 || !clock2)
12397                 return false;
12398
12399         diff = abs(clock1 - clock2);
12400
12401         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12402                 return true;
12403
12404         return false;
12405 }
12406
12407 static bool
12408 intel_compare_m_n(unsigned int m, unsigned int n,
12409                   unsigned int m2, unsigned int n2,
12410                   bool exact)
12411 {
12412         if (m == m2 && n == n2)
12413                 return true;
12414
12415         if (exact || !m || !n || !m2 || !n2)
12416                 return false;
12417
12418         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12419
12420         if (n > n2) {
12421                 while (n > n2) {
12422                         m2 <<= 1;
12423                         n2 <<= 1;
12424                 }
12425         } else if (n < n2) {
12426                 while (n < n2) {
12427                         m <<= 1;
12428                         n <<= 1;
12429                 }
12430         }
12431
12432         if (n != n2)
12433                 return false;
12434
12435         return intel_fuzzy_clock_check(m, m2);
12436 }
12437
12438 static bool
12439 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12440                        const struct intel_link_m_n *m2_n2,
12441                        bool exact)
12442 {
12443         return m_n->tu == m2_n2->tu &&
12444                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12445                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
12446                 intel_compare_m_n(m_n->link_m, m_n->link_n,
12447                                   m2_n2->link_m, m2_n2->link_n, exact);
12448 }
12449
12450 static bool
12451 intel_compare_infoframe(const union hdmi_infoframe *a,
12452                         const union hdmi_infoframe *b)
12453 {
12454         return memcmp(a, b, sizeof(*a)) == 0;
12455 }
12456
12457 static void
12458 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
12459                                bool fastset, const char *name,
12460                                const union hdmi_infoframe *a,
12461                                const union hdmi_infoframe *b)
12462 {
12463         if (fastset) {
12464                 if ((drm_debug & DRM_UT_KMS) == 0)
12465                         return;
12466
12467                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s infoframe", name);
12468                 drm_dbg(DRM_UT_KMS, "expected:");
12469                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
12470                 drm_dbg(DRM_UT_KMS, "found");
12471                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
12472         } else {
12473                 drm_err("mismatch in %s infoframe", name);
12474                 drm_err("expected:");
12475                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
12476                 drm_err("found");
12477                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
12478         }
12479 }
12480
12481 static void __printf(3, 4)
12482 pipe_config_mismatch(bool fastset, const char *name, const char *format, ...)
12483 {
12484         struct va_format vaf;
12485         va_list args;
12486
12487         va_start(args, format);
12488         vaf.fmt = format;
12489         vaf.va = &args;
12490
12491         if (fastset)
12492                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s %pV", name, &vaf);
12493         else
12494                 drm_err("mismatch in %s %pV", name, &vaf);
12495
12496         va_end(args);
12497 }
12498
12499 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
12500 {
12501         if (i915_modparams.fastboot != -1)
12502                 return i915_modparams.fastboot;
12503
12504         /* Enable fastboot by default on Skylake and newer */
12505         if (INTEL_GEN(dev_priv) >= 9)
12506                 return true;
12507
12508         /* Enable fastboot by default on VLV and CHV */
12509         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12510                 return true;
12511
12512         /* Disabled by default on all others */
12513         return false;
12514 }
12515
12516 static bool
12517 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
12518                           const struct intel_crtc_state *pipe_config,
12519                           bool fastset)
12520 {
12521         struct drm_i915_private *dev_priv = to_i915(current_config->base.crtc->dev);
12522         bool ret = true;
12523         bool fixup_inherited = fastset &&
12524                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
12525                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
12526
12527         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
12528                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
12529                 ret = false;
12530         }
12531
12532 #define PIPE_CONF_CHECK_X(name) do { \
12533         if (current_config->name != pipe_config->name) { \
12534                 pipe_config_mismatch(fastset, __stringify(name), \
12535                                      "(expected 0x%08x, found 0x%08x)\n", \
12536                                      current_config->name, \
12537                                      pipe_config->name); \
12538                 ret = false; \
12539         } \
12540 } while (0)
12541
12542 #define PIPE_CONF_CHECK_I(name) do { \
12543         if (current_config->name != pipe_config->name) { \
12544                 pipe_config_mismatch(fastset, __stringify(name), \
12545                                      "(expected %i, found %i)\n", \
12546                                      current_config->name, \
12547                                      pipe_config->name); \
12548                 ret = false; \
12549         } \
12550 } while (0)
12551
12552 #define PIPE_CONF_CHECK_BOOL(name) do { \
12553         if (current_config->name != pipe_config->name) { \
12554                 pipe_config_mismatch(fastset, __stringify(name), \
12555                                      "(expected %s, found %s)\n", \
12556                                      yesno(current_config->name), \
12557                                      yesno(pipe_config->name)); \
12558                 ret = false; \
12559         } \
12560 } while (0)
12561
12562 /*
12563  * Checks state where we only read out the enabling, but not the entire
12564  * state itself (like full infoframes or ELD for audio). These states
12565  * require a full modeset on bootup to fix up.
12566  */
12567 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
12568         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
12569                 PIPE_CONF_CHECK_BOOL(name); \
12570         } else { \
12571                 pipe_config_mismatch(fastset, __stringify(name), \
12572                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
12573                                      yesno(current_config->name), \
12574                                      yesno(pipe_config->name)); \
12575                 ret = false; \
12576         } \
12577 } while (0)
12578
12579 #define PIPE_CONF_CHECK_P(name) do { \
12580         if (current_config->name != pipe_config->name) { \
12581                 pipe_config_mismatch(fastset, __stringify(name), \
12582                                      "(expected %p, found %p)\n", \
12583                                      current_config->name, \
12584                                      pipe_config->name); \
12585                 ret = false; \
12586         } \
12587 } while (0)
12588
12589 #define PIPE_CONF_CHECK_M_N(name) do { \
12590         if (!intel_compare_link_m_n(&current_config->name, \
12591                                     &pipe_config->name,\
12592                                     !fastset)) { \
12593                 pipe_config_mismatch(fastset, __stringify(name), \
12594                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12595                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12596                                      current_config->name.tu, \
12597                                      current_config->name.gmch_m, \
12598                                      current_config->name.gmch_n, \
12599                                      current_config->name.link_m, \
12600                                      current_config->name.link_n, \
12601                                      pipe_config->name.tu, \
12602                                      pipe_config->name.gmch_m, \
12603                                      pipe_config->name.gmch_n, \
12604                                      pipe_config->name.link_m, \
12605                                      pipe_config->name.link_n); \
12606                 ret = false; \
12607         } \
12608 } while (0)
12609
12610 /* This is required for BDW+ where there is only one set of registers for
12611  * switching between high and low RR.
12612  * This macro can be used whenever a comparison has to be made between one
12613  * hw state and multiple sw state variables.
12614  */
12615 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
12616         if (!intel_compare_link_m_n(&current_config->name, \
12617                                     &pipe_config->name, !fastset) && \
12618             !intel_compare_link_m_n(&current_config->alt_name, \
12619                                     &pipe_config->name, !fastset)) { \
12620                 pipe_config_mismatch(fastset, __stringify(name), \
12621                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12622                                      "or tu %i gmch %i/%i link %i/%i, " \
12623                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12624                                      current_config->name.tu, \
12625                                      current_config->name.gmch_m, \
12626                                      current_config->name.gmch_n, \
12627                                      current_config->name.link_m, \
12628                                      current_config->name.link_n, \
12629                                      current_config->alt_name.tu, \
12630                                      current_config->alt_name.gmch_m, \
12631                                      current_config->alt_name.gmch_n, \
12632                                      current_config->alt_name.link_m, \
12633                                      current_config->alt_name.link_n, \
12634                                      pipe_config->name.tu, \
12635                                      pipe_config->name.gmch_m, \
12636                                      pipe_config->name.gmch_n, \
12637                                      pipe_config->name.link_m, \
12638                                      pipe_config->name.link_n); \
12639                 ret = false; \
12640         } \
12641 } while (0)
12642
12643 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
12644         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12645                 pipe_config_mismatch(fastset, __stringify(name), \
12646                                      "(%x) (expected %i, found %i)\n", \
12647                                      (mask), \
12648                                      current_config->name & (mask), \
12649                                      pipe_config->name & (mask)); \
12650                 ret = false; \
12651         } \
12652 } while (0)
12653
12654 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
12655         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12656                 pipe_config_mismatch(fastset, __stringify(name), \
12657                                      "(expected %i, found %i)\n", \
12658                                      current_config->name, \
12659                                      pipe_config->name); \
12660                 ret = false; \
12661         } \
12662 } while (0)
12663
12664 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
12665         if (!intel_compare_infoframe(&current_config->infoframes.name, \
12666                                      &pipe_config->infoframes.name)) { \
12667                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
12668                                                &current_config->infoframes.name, \
12669                                                &pipe_config->infoframes.name); \
12670                 ret = false; \
12671         } \
12672 } while (0)
12673
12674 #define PIPE_CONF_QUIRK(quirk) \
12675         ((current_config->quirks | pipe_config->quirks) & (quirk))
12676
12677         PIPE_CONF_CHECK_I(cpu_transcoder);
12678
12679         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
12680         PIPE_CONF_CHECK_I(fdi_lanes);
12681         PIPE_CONF_CHECK_M_N(fdi_m_n);
12682
12683         PIPE_CONF_CHECK_I(lane_count);
12684         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
12685
12686         if (INTEL_GEN(dev_priv) < 8) {
12687                 PIPE_CONF_CHECK_M_N(dp_m_n);
12688
12689                 if (current_config->has_drrs)
12690                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12691         } else
12692                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12693
12694         PIPE_CONF_CHECK_X(output_types);
12695
12696         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12697         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12698         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12699         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12700         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12701         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12702
12703         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12704         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12705         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12706         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12707         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12708         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12709
12710         PIPE_CONF_CHECK_I(pixel_multiplier);
12711         PIPE_CONF_CHECK_I(output_format);
12712         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
12713         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
12714             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12715                 PIPE_CONF_CHECK_BOOL(limited_color_range);
12716
12717         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
12718         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
12719         PIPE_CONF_CHECK_BOOL(has_infoframe);
12720
12721         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
12722
12723         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12724                               DRM_MODE_FLAG_INTERLACE);
12725
12726         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12727                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12728                                       DRM_MODE_FLAG_PHSYNC);
12729                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12730                                       DRM_MODE_FLAG_NHSYNC);
12731                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12732                                       DRM_MODE_FLAG_PVSYNC);
12733                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12734                                       DRM_MODE_FLAG_NVSYNC);
12735         }
12736
12737         PIPE_CONF_CHECK_X(gmch_pfit.control);
12738         /* pfit ratios are autocomputed by the hw on gen4+ */
12739         if (INTEL_GEN(dev_priv) < 4)
12740                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
12741         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12742
12743         /*
12744          * Changing the EDP transcoder input mux
12745          * (A_ONOFF vs. A_ON) requires a full modeset.
12746          */
12747         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
12748
12749         if (!fastset) {
12750                 PIPE_CONF_CHECK_I(pipe_src_w);
12751                 PIPE_CONF_CHECK_I(pipe_src_h);
12752
12753                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
12754                 if (current_config->pch_pfit.enabled) {
12755                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12756                         PIPE_CONF_CHECK_X(pch_pfit.size);
12757                 }
12758
12759                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12760                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
12761
12762                 PIPE_CONF_CHECK_X(gamma_mode);
12763                 if (IS_CHERRYVIEW(dev_priv))
12764                         PIPE_CONF_CHECK_X(cgm_mode);
12765                 else
12766                         PIPE_CONF_CHECK_X(csc_mode);
12767                 PIPE_CONF_CHECK_BOOL(gamma_enable);
12768                 PIPE_CONF_CHECK_BOOL(csc_enable);
12769         }
12770
12771         PIPE_CONF_CHECK_BOOL(double_wide);
12772
12773         PIPE_CONF_CHECK_P(shared_dpll);
12774         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12775         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12776         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12777         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12778         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12779         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12780         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12781         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12782         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12783         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
12784         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
12785         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
12786         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
12787         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
12788         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
12789         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
12790         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
12791         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
12792         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
12793         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
12794         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
12795         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
12796         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
12797         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
12798         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
12799         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
12800         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
12801         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
12802         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
12803         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
12804         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
12805
12806         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
12807         PIPE_CONF_CHECK_X(dsi_pll.div);
12808
12809         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
12810                 PIPE_CONF_CHECK_I(pipe_bpp);
12811
12812         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12813         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12814
12815         PIPE_CONF_CHECK_I(min_voltage_level);
12816
12817         PIPE_CONF_CHECK_X(infoframes.enable);
12818         PIPE_CONF_CHECK_X(infoframes.gcp);
12819         PIPE_CONF_CHECK_INFOFRAME(avi);
12820         PIPE_CONF_CHECK_INFOFRAME(spd);
12821         PIPE_CONF_CHECK_INFOFRAME(hdmi);
12822         PIPE_CONF_CHECK_INFOFRAME(drm);
12823
12824 #undef PIPE_CONF_CHECK_X
12825 #undef PIPE_CONF_CHECK_I
12826 #undef PIPE_CONF_CHECK_BOOL
12827 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
12828 #undef PIPE_CONF_CHECK_P
12829 #undef PIPE_CONF_CHECK_FLAGS
12830 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12831 #undef PIPE_CONF_QUIRK
12832
12833         return ret;
12834 }
12835
12836 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
12837                                            const struct intel_crtc_state *pipe_config)
12838 {
12839         if (pipe_config->has_pch_encoder) {
12840                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12841                                                             &pipe_config->fdi_m_n);
12842                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
12843
12844                 /*
12845                  * FDI already provided one idea for the dotclock.
12846                  * Yell if the encoder disagrees.
12847                  */
12848                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
12849                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12850                      fdi_dotclock, dotclock);
12851         }
12852 }
12853
12854 static void verify_wm_state(struct intel_crtc *crtc,
12855                             struct intel_crtc_state *new_crtc_state)
12856 {
12857         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12858         struct skl_hw_state {
12859                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
12860                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
12861                 struct skl_ddb_allocation ddb;
12862                 struct skl_pipe_wm wm;
12863         } *hw;
12864         struct skl_ddb_allocation *sw_ddb;
12865         struct skl_pipe_wm *sw_wm;
12866         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
12867         const enum pipe pipe = crtc->pipe;
12868         int plane, level, max_level = ilk_wm_max_level(dev_priv);
12869
12870         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->base.active)
12871                 return;
12872
12873         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
12874         if (!hw)
12875                 return;
12876
12877         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
12878         sw_wm = &new_crtc_state->wm.skl.optimal;
12879
12880         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
12881
12882         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
12883         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12884
12885         if (INTEL_GEN(dev_priv) >= 11 &&
12886             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
12887                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
12888                           sw_ddb->enabled_slices,
12889                           hw->ddb.enabled_slices);
12890
12891         /* planes */
12892         for_each_universal_plane(dev_priv, pipe, plane) {
12893                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12894
12895                 hw_plane_wm = &hw->wm.planes[plane];
12896                 sw_plane_wm = &sw_wm->planes[plane];
12897
12898                 /* Watermarks */
12899                 for (level = 0; level <= max_level; level++) {
12900                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12901                                                 &sw_plane_wm->wm[level]))
12902                                 continue;
12903
12904                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12905                                   pipe_name(pipe), plane + 1, level,
12906                                   sw_plane_wm->wm[level].plane_en,
12907                                   sw_plane_wm->wm[level].plane_res_b,
12908                                   sw_plane_wm->wm[level].plane_res_l,
12909                                   hw_plane_wm->wm[level].plane_en,
12910                                   hw_plane_wm->wm[level].plane_res_b,
12911                                   hw_plane_wm->wm[level].plane_res_l);
12912                 }
12913
12914                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12915                                          &sw_plane_wm->trans_wm)) {
12916                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12917                                   pipe_name(pipe), plane + 1,
12918                                   sw_plane_wm->trans_wm.plane_en,
12919                                   sw_plane_wm->trans_wm.plane_res_b,
12920                                   sw_plane_wm->trans_wm.plane_res_l,
12921                                   hw_plane_wm->trans_wm.plane_en,
12922                                   hw_plane_wm->trans_wm.plane_res_b,
12923                                   hw_plane_wm->trans_wm.plane_res_l);
12924                 }
12925
12926                 /* DDB */
12927                 hw_ddb_entry = &hw->ddb_y[plane];
12928                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
12929
12930                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12931                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12932                                   pipe_name(pipe), plane + 1,
12933                                   sw_ddb_entry->start, sw_ddb_entry->end,
12934                                   hw_ddb_entry->start, hw_ddb_entry->end);
12935                 }
12936         }
12937
12938         /*
12939          * cursor
12940          * If the cursor plane isn't active, we may not have updated it's ddb
12941          * allocation. In that case since the ddb allocation will be updated
12942          * once the plane becomes visible, we can skip this check
12943          */
12944         if (1) {
12945                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12946
12947                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
12948                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12949
12950                 /* Watermarks */
12951                 for (level = 0; level <= max_level; level++) {
12952                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12953                                                 &sw_plane_wm->wm[level]))
12954                                 continue;
12955
12956                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12957                                   pipe_name(pipe), level,
12958                                   sw_plane_wm->wm[level].plane_en,
12959                                   sw_plane_wm->wm[level].plane_res_b,
12960                                   sw_plane_wm->wm[level].plane_res_l,
12961                                   hw_plane_wm->wm[level].plane_en,
12962                                   hw_plane_wm->wm[level].plane_res_b,
12963                                   hw_plane_wm->wm[level].plane_res_l);
12964                 }
12965
12966                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12967                                          &sw_plane_wm->trans_wm)) {
12968                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12969                                   pipe_name(pipe),
12970                                   sw_plane_wm->trans_wm.plane_en,
12971                                   sw_plane_wm->trans_wm.plane_res_b,
12972                                   sw_plane_wm->trans_wm.plane_res_l,
12973                                   hw_plane_wm->trans_wm.plane_en,
12974                                   hw_plane_wm->trans_wm.plane_res_b,
12975                                   hw_plane_wm->trans_wm.plane_res_l);
12976                 }
12977
12978                 /* DDB */
12979                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
12980                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
12981
12982                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12983                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
12984                                   pipe_name(pipe),
12985                                   sw_ddb_entry->start, sw_ddb_entry->end,
12986                                   hw_ddb_entry->start, hw_ddb_entry->end);
12987                 }
12988         }
12989
12990         kfree(hw);
12991 }
12992
12993 static void
12994 verify_connector_state(struct intel_atomic_state *state,
12995                        struct intel_crtc *crtc)
12996 {
12997         struct drm_connector *connector;
12998         struct drm_connector_state *new_conn_state;
12999         int i;
13000
13001         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13002                 struct drm_encoder *encoder = connector->encoder;
13003                 struct intel_crtc_state *crtc_state = NULL;
13004
13005                 if (new_conn_state->crtc != &crtc->base)
13006                         continue;
13007
13008                 if (crtc)
13009                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13010
13011                 intel_connector_verify_state(crtc_state, new_conn_state);
13012
13013                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13014                      "connector's atomic encoder doesn't match legacy encoder\n");
13015         }
13016 }
13017
13018 static void
13019 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13020 {
13021         struct intel_encoder *encoder;
13022         struct drm_connector *connector;
13023         struct drm_connector_state *old_conn_state, *new_conn_state;
13024         int i;
13025
13026         for_each_intel_encoder(&dev_priv->drm, encoder) {
13027                 bool enabled = false, found = false;
13028                 enum pipe pipe;
13029
13030                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13031                               encoder->base.base.id,
13032                               encoder->base.name);
13033
13034                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13035                                                    new_conn_state, i) {
13036                         if (old_conn_state->best_encoder == &encoder->base)
13037                                 found = true;
13038
13039                         if (new_conn_state->best_encoder != &encoder->base)
13040                                 continue;
13041                         found = enabled = true;
13042
13043                         I915_STATE_WARN(new_conn_state->crtc !=
13044                                         encoder->base.crtc,
13045                              "connector's crtc doesn't match encoder crtc\n");
13046                 }
13047
13048                 if (!found)
13049                         continue;
13050
13051                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13052                      "encoder's enabled state mismatch "
13053                      "(expected %i, found %i)\n",
13054                      !!encoder->base.crtc, enabled);
13055
13056                 if (!encoder->base.crtc) {
13057                         bool active;
13058
13059                         active = encoder->get_hw_state(encoder, &pipe);
13060                         I915_STATE_WARN(active,
13061                              "encoder detached but still enabled on pipe %c.\n",
13062                              pipe_name(pipe));
13063                 }
13064         }
13065 }
13066
13067 static void
13068 verify_crtc_state(struct intel_crtc *crtc,
13069                   struct intel_crtc_state *old_crtc_state,
13070                   struct intel_crtc_state *new_crtc_state)
13071 {
13072         struct drm_device *dev = crtc->base.dev;
13073         struct drm_i915_private *dev_priv = to_i915(dev);
13074         struct intel_encoder *encoder;
13075         struct intel_crtc_state *pipe_config;
13076         struct drm_atomic_state *state;
13077         bool active;
13078
13079         state = old_crtc_state->base.state;
13080         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->base);
13081         pipe_config = old_crtc_state;
13082         memset(pipe_config, 0, sizeof(*pipe_config));
13083         pipe_config->base.crtc = &crtc->base;
13084         pipe_config->base.state = state;
13085
13086         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13087
13088         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13089
13090         /* we keep both pipes enabled on 830 */
13091         if (IS_I830(dev_priv))
13092                 active = new_crtc_state->base.active;
13093
13094         I915_STATE_WARN(new_crtc_state->base.active != active,
13095              "crtc active state doesn't match with hw state "
13096              "(expected %i, found %i)\n", new_crtc_state->base.active, active);
13097
13098         I915_STATE_WARN(crtc->active != new_crtc_state->base.active,
13099              "transitional active state does not match atomic hw state "
13100              "(expected %i, found %i)\n", new_crtc_state->base.active, crtc->active);
13101
13102         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13103                 enum pipe pipe;
13104
13105                 active = encoder->get_hw_state(encoder, &pipe);
13106                 I915_STATE_WARN(active != new_crtc_state->base.active,
13107                         "[ENCODER:%i] active %i with crtc active %i\n",
13108                         encoder->base.base.id, active, new_crtc_state->base.active);
13109
13110                 I915_STATE_WARN(active && crtc->pipe != pipe,
13111                                 "Encoder connected to wrong pipe %c\n",
13112                                 pipe_name(pipe));
13113
13114                 if (active)
13115                         encoder->get_config(encoder, pipe_config);
13116         }
13117
13118         intel_crtc_compute_pixel_rate(pipe_config);
13119
13120         if (!new_crtc_state->base.active)
13121                 return;
13122
13123         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13124
13125         if (!intel_pipe_config_compare(new_crtc_state,
13126                                        pipe_config, false)) {
13127                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13128                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13129                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13130         }
13131 }
13132
13133 static void
13134 intel_verify_planes(struct intel_atomic_state *state)
13135 {
13136         struct intel_plane *plane;
13137         const struct intel_plane_state *plane_state;
13138         int i;
13139
13140         for_each_new_intel_plane_in_state(state, plane,
13141                                           plane_state, i)
13142                 assert_plane(plane, plane_state->slave ||
13143                              plane_state->base.visible);
13144 }
13145
13146 static void
13147 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13148                          struct intel_shared_dpll *pll,
13149                          struct intel_crtc *crtc,
13150                          struct intel_crtc_state *new_crtc_state)
13151 {
13152         struct intel_dpll_hw_state dpll_hw_state;
13153         unsigned int crtc_mask;
13154         bool active;
13155
13156         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13157
13158         DRM_DEBUG_KMS("%s\n", pll->info->name);
13159
13160         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13161
13162         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13163                 I915_STATE_WARN(!pll->on && pll->active_mask,
13164                      "pll in active use but not on in sw tracking\n");
13165                 I915_STATE_WARN(pll->on && !pll->active_mask,
13166                      "pll is on but not used by any active crtc\n");
13167                 I915_STATE_WARN(pll->on != active,
13168                      "pll on state mismatch (expected %i, found %i)\n",
13169                      pll->on, active);
13170         }
13171
13172         if (!crtc) {
13173                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13174                                 "more active pll users than references: %x vs %x\n",
13175                                 pll->active_mask, pll->state.crtc_mask);
13176
13177                 return;
13178         }
13179
13180         crtc_mask = drm_crtc_mask(&crtc->base);
13181
13182         if (new_crtc_state->base.active)
13183                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13184                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13185                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13186         else
13187                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13188                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13189                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13190
13191         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13192                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13193                         crtc_mask, pll->state.crtc_mask);
13194
13195         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13196                                           &dpll_hw_state,
13197                                           sizeof(dpll_hw_state)),
13198                         "pll hw state mismatch\n");
13199 }
13200
13201 static void
13202 verify_shared_dpll_state(struct intel_crtc *crtc,
13203                          struct intel_crtc_state *old_crtc_state,
13204                          struct intel_crtc_state *new_crtc_state)
13205 {
13206         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13207
13208         if (new_crtc_state->shared_dpll)
13209                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13210
13211         if (old_crtc_state->shared_dpll &&
13212             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13213                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13214                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13215
13216                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13217                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13218                                 pipe_name(drm_crtc_index(&crtc->base)));
13219                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13220                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13221                                 pipe_name(drm_crtc_index(&crtc->base)));
13222         }
13223 }
13224
13225 static void
13226 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13227                           struct intel_atomic_state *state,
13228                           struct intel_crtc_state *old_crtc_state,
13229                           struct intel_crtc_state *new_crtc_state)
13230 {
13231         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13232                 return;
13233
13234         verify_wm_state(crtc, new_crtc_state);
13235         verify_connector_state(state, crtc);
13236         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13237         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13238 }
13239
13240 static void
13241 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13242 {
13243         int i;
13244
13245         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13246                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13247 }
13248
13249 static void
13250 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13251                               struct intel_atomic_state *state)
13252 {
13253         verify_encoder_state(dev_priv, state);
13254         verify_connector_state(state, NULL);
13255         verify_disabled_dpll_state(dev_priv);
13256 }
13257
13258 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
13259 {
13260         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13261         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13262
13263         /*
13264          * The scanline counter increments at the leading edge of hsync.
13265          *
13266          * On most platforms it starts counting from vtotal-1 on the
13267          * first active line. That means the scanline counter value is
13268          * always one less than what we would expect. Ie. just after
13269          * start of vblank, which also occurs at start of hsync (on the
13270          * last active line), the scanline counter will read vblank_start-1.
13271          *
13272          * On gen2 the scanline counter starts counting from 1 instead
13273          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13274          * to keep the value positive), instead of adding one.
13275          *
13276          * On HSW+ the behaviour of the scanline counter depends on the output
13277          * type. For DP ports it behaves like most other platforms, but on HDMI
13278          * there's an extra 1 line difference. So we need to add two instead of
13279          * one to the value.
13280          *
13281          * On VLV/CHV DSI the scanline counter would appear to increment
13282          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13283          * that means we can't tell whether we're in vblank or not while
13284          * we're on that particular line. We must still set scanline_offset
13285          * to 1 so that the vblank timestamps come out correct when we query
13286          * the scanline counter from within the vblank interrupt handler.
13287          * However if queried just before the start of vblank we'll get an
13288          * answer that's slightly in the future.
13289          */
13290         if (IS_GEN(dev_priv, 2)) {
13291                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
13292                 int vtotal;
13293
13294                 vtotal = adjusted_mode->crtc_vtotal;
13295                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13296                         vtotal /= 2;
13297
13298                 crtc->scanline_offset = vtotal - 1;
13299         } else if (HAS_DDI(dev_priv) &&
13300                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
13301                 crtc->scanline_offset = 2;
13302         } else
13303                 crtc->scanline_offset = 1;
13304 }
13305
13306 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
13307 {
13308         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13309         struct intel_crtc_state *new_crtc_state;
13310         struct intel_crtc *crtc;
13311         int i;
13312
13313         if (!dev_priv->display.crtc_compute_clock)
13314                 return;
13315
13316         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13317                 if (!needs_modeset(new_crtc_state))
13318                         continue;
13319
13320                 intel_release_shared_dplls(state, crtc);
13321         }
13322 }
13323
13324 /*
13325  * This implements the workaround described in the "notes" section of the mode
13326  * set sequence documentation. When going from no pipes or single pipe to
13327  * multiple pipes, and planes are enabled after the pipe, we need to wait at
13328  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
13329  */
13330 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
13331 {
13332         struct intel_crtc_state *crtc_state;
13333         struct intel_crtc *crtc;
13334         struct intel_crtc_state *first_crtc_state = NULL;
13335         struct intel_crtc_state *other_crtc_state = NULL;
13336         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13337         int i;
13338
13339         /* look at all crtc's that are going to be enabled in during modeset */
13340         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
13341                 if (!crtc_state->base.active ||
13342                     !needs_modeset(crtc_state))
13343                         continue;
13344
13345                 if (first_crtc_state) {
13346                         other_crtc_state = crtc_state;
13347                         break;
13348                 } else {
13349                         first_crtc_state = crtc_state;
13350                         first_pipe = crtc->pipe;
13351                 }
13352         }
13353
13354         /* No workaround needed? */
13355         if (!first_crtc_state)
13356                 return 0;
13357
13358         /* w/a possibly needed, check how many crtc's are already enabled. */
13359         for_each_intel_crtc(state->base.dev, crtc) {
13360                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13361                 if (IS_ERR(crtc_state))
13362                         return PTR_ERR(crtc_state);
13363
13364                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
13365
13366                 if (!crtc_state->base.active ||
13367                     needs_modeset(crtc_state))
13368                         continue;
13369
13370                 /* 2 or more enabled crtcs means no need for w/a */
13371                 if (enabled_pipe != INVALID_PIPE)
13372                         return 0;
13373
13374                 enabled_pipe = crtc->pipe;
13375         }
13376
13377         if (enabled_pipe != INVALID_PIPE)
13378                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13379         else if (other_crtc_state)
13380                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13381
13382         return 0;
13383 }
13384
13385 static int intel_lock_all_pipes(struct drm_atomic_state *state)
13386 {
13387         struct drm_crtc *crtc;
13388
13389         /* Add all pipes to the state */
13390         for_each_crtc(state->dev, crtc) {
13391                 struct drm_crtc_state *crtc_state;
13392
13393                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13394                 if (IS_ERR(crtc_state))
13395                         return PTR_ERR(crtc_state);
13396         }
13397
13398         return 0;
13399 }
13400
13401 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
13402 {
13403         struct drm_crtc *crtc;
13404
13405         /*
13406          * Add all pipes to the state, and force
13407          * a modeset on all the active ones.
13408          */
13409         for_each_crtc(state->dev, crtc) {
13410                 struct drm_crtc_state *crtc_state;
13411                 int ret;
13412
13413                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13414                 if (IS_ERR(crtc_state))
13415                         return PTR_ERR(crtc_state);
13416
13417                 if (!crtc_state->active || needs_modeset(to_intel_crtc_state(crtc_state)))
13418                         continue;
13419
13420                 crtc_state->mode_changed = true;
13421
13422                 ret = drm_atomic_add_affected_connectors(state, crtc);
13423                 if (ret)
13424                         return ret;
13425
13426                 ret = drm_atomic_add_affected_planes(state, crtc);
13427                 if (ret)
13428                         return ret;
13429         }
13430
13431         return 0;
13432 }
13433
13434 static int intel_modeset_checks(struct intel_atomic_state *state)
13435 {
13436         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13437         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13438         struct intel_crtc *crtc;
13439         int ret = 0, i;
13440
13441         if (!check_digital_port_conflicts(state)) {
13442                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13443                 return -EINVAL;
13444         }
13445
13446         /* keep the current setting */
13447         if (!state->cdclk.force_min_cdclk_changed)
13448                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
13449
13450         state->modeset = true;
13451         state->active_crtcs = dev_priv->active_crtcs;
13452         state->cdclk.logical = dev_priv->cdclk.logical;
13453         state->cdclk.actual = dev_priv->cdclk.actual;
13454         state->cdclk.pipe = INVALID_PIPE;
13455
13456         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13457                                             new_crtc_state, i) {
13458                 if (new_crtc_state->base.active)
13459                         state->active_crtcs |= 1 << i;
13460                 else
13461                         state->active_crtcs &= ~(1 << i);
13462
13463                 if (old_crtc_state->base.active != new_crtc_state->base.active)
13464                         state->active_pipe_changes |= drm_crtc_mask(&crtc->base);
13465         }
13466
13467         /*
13468          * See if the config requires any additional preparation, e.g.
13469          * to adjust global state with pipes off.  We need to do this
13470          * here so we can get the modeset_pipe updated config for the new
13471          * mode set on this crtc.  For other crtcs we need to use the
13472          * adjusted_mode bits in the crtc directly.
13473          */
13474         if (dev_priv->display.modeset_calc_cdclk) {
13475                 enum pipe pipe;
13476
13477                 ret = dev_priv->display.modeset_calc_cdclk(state);
13478                 if (ret < 0)
13479                         return ret;
13480
13481                 /*
13482                  * Writes to dev_priv->cdclk.logical must protected by
13483                  * holding all the crtc locks, even if we don't end up
13484                  * touching the hardware
13485                  */
13486                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
13487                                         &state->cdclk.logical)) {
13488                         ret = intel_lock_all_pipes(&state->base);
13489                         if (ret < 0)
13490                                 return ret;
13491                 }
13492
13493                 if (is_power_of_2(state->active_crtcs)) {
13494                         struct intel_crtc *crtc;
13495                         struct intel_crtc_state *crtc_state;
13496
13497                         pipe = ilog2(state->active_crtcs);
13498                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
13499                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13500                         if (crtc_state && needs_modeset(crtc_state))
13501                                 pipe = INVALID_PIPE;
13502                 } else {
13503                         pipe = INVALID_PIPE;
13504                 }
13505
13506                 /* All pipes must be switched off while we change the cdclk. */
13507                 if (pipe != INVALID_PIPE &&
13508                     intel_cdclk_needs_cd2x_update(dev_priv,
13509                                                   &dev_priv->cdclk.actual,
13510                                                   &state->cdclk.actual)) {
13511                         ret = intel_lock_all_pipes(&state->base);
13512                         if (ret < 0)
13513                                 return ret;
13514
13515                         state->cdclk.pipe = pipe;
13516                 } else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
13517                                                      &state->cdclk.actual)) {
13518                         ret = intel_modeset_all_pipes(&state->base);
13519                         if (ret < 0)
13520                                 return ret;
13521
13522                         state->cdclk.pipe = INVALID_PIPE;
13523                 }
13524
13525                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
13526                               state->cdclk.logical.cdclk,
13527                               state->cdclk.actual.cdclk);
13528                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
13529                               state->cdclk.logical.voltage_level,
13530                               state->cdclk.actual.voltage_level);
13531         }
13532
13533         intel_modeset_clear_plls(state);
13534
13535         if (IS_HASWELL(dev_priv))
13536                 return haswell_mode_set_planes_workaround(state);
13537
13538         return 0;
13539 }
13540
13541 /*
13542  * Handle calculation of various watermark data at the end of the atomic check
13543  * phase.  The code here should be run after the per-crtc and per-plane 'check'
13544  * handlers to ensure that all derived state has been updated.
13545  */
13546 static int calc_watermark_data(struct intel_atomic_state *state)
13547 {
13548         struct drm_device *dev = state->base.dev;
13549         struct drm_i915_private *dev_priv = to_i915(dev);
13550
13551         /* Is there platform-specific watermark information to calculate? */
13552         if (dev_priv->display.compute_global_watermarks)
13553                 return dev_priv->display.compute_global_watermarks(state);
13554
13555         return 0;
13556 }
13557
13558 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
13559                                      struct intel_crtc_state *new_crtc_state)
13560 {
13561         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
13562                 return;
13563
13564         new_crtc_state->base.mode_changed = false;
13565         new_crtc_state->update_pipe = true;
13566
13567         /*
13568          * If we're not doing the full modeset we want to
13569          * keep the current M/N values as they may be
13570          * sufficiently different to the computed values
13571          * to cause problems.
13572          *
13573          * FIXME: should really copy more fuzzy state here
13574          */
13575         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
13576         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
13577         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
13578         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
13579 }
13580
13581 /**
13582  * intel_atomic_check - validate state object
13583  * @dev: drm device
13584  * @_state: state to validate
13585  */
13586 static int intel_atomic_check(struct drm_device *dev,
13587                               struct drm_atomic_state *_state)
13588 {
13589         struct drm_i915_private *dev_priv = to_i915(dev);
13590         struct intel_atomic_state *state = to_intel_atomic_state(_state);
13591         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13592         struct intel_crtc *crtc;
13593         int ret, i;
13594         bool any_ms = state->cdclk.force_min_cdclk_changed;
13595
13596         /* Catch I915_MODE_FLAG_INHERITED */
13597         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13598                                             new_crtc_state, i) {
13599                 if (new_crtc_state->base.mode.private_flags !=
13600                     old_crtc_state->base.mode.private_flags)
13601                         new_crtc_state->base.mode_changed = true;
13602         }
13603
13604         ret = drm_atomic_helper_check_modeset(dev, &state->base);
13605         if (ret)
13606                 goto fail;
13607
13608         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13609                                             new_crtc_state, i) {
13610                 if (!needs_modeset(new_crtc_state))
13611                         continue;
13612
13613                 if (!new_crtc_state->base.enable) {
13614                         any_ms = true;
13615                         continue;
13616                 }
13617
13618                 ret = intel_modeset_pipe_config(new_crtc_state);
13619                 if (ret)
13620                         goto fail;
13621
13622                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
13623
13624                 if (needs_modeset(new_crtc_state))
13625                         any_ms = true;
13626         }
13627
13628         ret = drm_dp_mst_atomic_check(&state->base);
13629         if (ret)
13630                 goto fail;
13631
13632         if (any_ms) {
13633                 ret = intel_modeset_checks(state);
13634                 if (ret)
13635                         goto fail;
13636         } else {
13637                 state->cdclk.logical = dev_priv->cdclk.logical;
13638         }
13639
13640         ret = icl_add_linked_planes(state);
13641         if (ret)
13642                 goto fail;
13643
13644         ret = drm_atomic_helper_check_planes(dev, &state->base);
13645         if (ret)
13646                 goto fail;
13647
13648         intel_fbc_choose_crtc(dev_priv, state);
13649         ret = calc_watermark_data(state);
13650         if (ret)
13651                 goto fail;
13652
13653         ret = intel_bw_atomic_check(state);
13654         if (ret)
13655                 goto fail;
13656
13657         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13658                                             new_crtc_state, i) {
13659                 if (!needs_modeset(new_crtc_state) &&
13660                     !new_crtc_state->update_pipe)
13661                         continue;
13662
13663                 intel_dump_pipe_config(new_crtc_state, state,
13664                                        needs_modeset(new_crtc_state) ?
13665                                        "[modeset]" : "[fastset]");
13666         }
13667
13668         return 0;
13669
13670  fail:
13671         if (ret == -EDEADLK)
13672                 return ret;
13673
13674         /*
13675          * FIXME would probably be nice to know which crtc specifically
13676          * caused the failure, in cases where we can pinpoint it.
13677          */
13678         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13679                                             new_crtc_state, i)
13680                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
13681
13682         return ret;
13683 }
13684
13685 static int intel_atomic_prepare_commit(struct drm_device *dev,
13686                                        struct drm_atomic_state *state)
13687 {
13688         return drm_atomic_helper_prepare_planes(dev, state);
13689 }
13690
13691 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
13692 {
13693         struct drm_device *dev = crtc->base.dev;
13694         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
13695
13696         if (!vblank->max_vblank_count)
13697                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
13698
13699         return crtc->base.funcs->get_vblank_counter(&crtc->base);
13700 }
13701
13702 static void intel_update_crtc(struct intel_crtc *crtc,
13703                               struct intel_atomic_state *state,
13704                               struct intel_crtc_state *old_crtc_state,
13705                               struct intel_crtc_state *new_crtc_state)
13706 {
13707         struct drm_device *dev = state->base.dev;
13708         struct drm_i915_private *dev_priv = to_i915(dev);
13709         bool modeset = needs_modeset(new_crtc_state);
13710         struct intel_plane_state *new_plane_state =
13711                 intel_atomic_get_new_plane_state(state,
13712                                                  to_intel_plane(crtc->base.primary));
13713
13714         if (modeset) {
13715                 update_scanline_offset(new_crtc_state);
13716                 dev_priv->display.crtc_enable(new_crtc_state, state);
13717
13718                 /* vblanks work again, re-enable pipe CRC. */
13719                 intel_crtc_enable_pipe_crc(crtc);
13720         } else {
13721                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13722
13723                 if (new_crtc_state->update_pipe)
13724                         intel_encoders_update_pipe(crtc, new_crtc_state, state);
13725         }
13726
13727         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
13728                 intel_fbc_disable(crtc);
13729         else if (new_plane_state)
13730                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
13731
13732         intel_begin_crtc_commit(state, crtc);
13733
13734         if (INTEL_GEN(dev_priv) >= 9)
13735                 skl_update_planes_on_crtc(state, crtc);
13736         else
13737                 i9xx_update_planes_on_crtc(state, crtc);
13738
13739         intel_finish_crtc_commit(state, crtc);
13740 }
13741
13742 static void intel_update_crtcs(struct intel_atomic_state *state)
13743 {
13744         struct intel_crtc *crtc;
13745         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13746         int i;
13747
13748         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13749                 if (!new_crtc_state->base.active)
13750                         continue;
13751
13752                 intel_update_crtc(crtc, state, old_crtc_state,
13753                                   new_crtc_state);
13754         }
13755 }
13756
13757 static void skl_update_crtcs(struct intel_atomic_state *state)
13758 {
13759         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13760         struct intel_crtc *crtc;
13761         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13762         unsigned int updated = 0;
13763         bool progress;
13764         enum pipe pipe;
13765         int i;
13766         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
13767         u8 required_slices = state->wm_results.ddb.enabled_slices;
13768         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
13769
13770         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
13771                 /* ignore allocations for crtc's that have been turned off. */
13772                 if (new_crtc_state->base.active)
13773                         entries[i] = old_crtc_state->wm.skl.ddb;
13774
13775         /* If 2nd DBuf slice required, enable it here */
13776         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
13777                 icl_dbuf_slices_update(dev_priv, required_slices);
13778
13779         /*
13780          * Whenever the number of active pipes changes, we need to make sure we
13781          * update the pipes in the right order so that their ddb allocations
13782          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
13783          * cause pipe underruns and other bad stuff.
13784          */
13785         do {
13786                 progress = false;
13787
13788                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13789                         bool vbl_wait = false;
13790                         unsigned int cmask = drm_crtc_mask(&crtc->base);
13791
13792                         pipe = crtc->pipe;
13793
13794                         if (updated & cmask || !new_crtc_state->base.active)
13795                                 continue;
13796
13797                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13798                                                         entries,
13799                                                         INTEL_INFO(dev_priv)->num_pipes, i))
13800                                 continue;
13801
13802                         updated |= cmask;
13803                         entries[i] = new_crtc_state->wm.skl.ddb;
13804
13805                         /*
13806                          * If this is an already active pipe, it's DDB changed,
13807                          * and this isn't the last pipe that needs updating
13808                          * then we need to wait for a vblank to pass for the
13809                          * new ddb allocation to take effect.
13810                          */
13811                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
13812                                                  &old_crtc_state->wm.skl.ddb) &&
13813                             !new_crtc_state->base.active_changed &&
13814                             state->wm_results.dirty_pipes != updated)
13815                                 vbl_wait = true;
13816
13817                         intel_update_crtc(crtc, state, old_crtc_state,
13818                                           new_crtc_state);
13819
13820                         if (vbl_wait)
13821                                 intel_wait_for_vblank(dev_priv, pipe);
13822
13823                         progress = true;
13824                 }
13825         } while (progress);
13826
13827         /* If 2nd DBuf slice is no more required disable it */
13828         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
13829                 icl_dbuf_slices_update(dev_priv, required_slices);
13830 }
13831
13832 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
13833 {
13834         struct intel_atomic_state *state, *next;
13835         struct llist_node *freed;
13836
13837         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
13838         llist_for_each_entry_safe(state, next, freed, freed)
13839                 drm_atomic_state_put(&state->base);
13840 }
13841
13842 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
13843 {
13844         struct drm_i915_private *dev_priv =
13845                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
13846
13847         intel_atomic_helper_free_state(dev_priv);
13848 }
13849
13850 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
13851 {
13852         struct wait_queue_entry wait_fence, wait_reset;
13853         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
13854
13855         init_wait_entry(&wait_fence, 0);
13856         init_wait_entry(&wait_reset, 0);
13857         for (;;) {
13858                 prepare_to_wait(&intel_state->commit_ready.wait,
13859                                 &wait_fence, TASK_UNINTERRUPTIBLE);
13860                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
13861                                 &wait_reset, TASK_UNINTERRUPTIBLE);
13862
13863
13864                 if (i915_sw_fence_done(&intel_state->commit_ready)
13865                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
13866                         break;
13867
13868                 schedule();
13869         }
13870         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
13871         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
13872 }
13873
13874 static void intel_atomic_cleanup_work(struct work_struct *work)
13875 {
13876         struct drm_atomic_state *state =
13877                 container_of(work, struct drm_atomic_state, commit_work);
13878         struct drm_i915_private *i915 = to_i915(state->dev);
13879
13880         drm_atomic_helper_cleanup_planes(&i915->drm, state);
13881         drm_atomic_helper_commit_cleanup_done(state);
13882         drm_atomic_state_put(state);
13883
13884         intel_atomic_helper_free_state(i915);
13885 }
13886
13887 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
13888 {
13889         struct drm_device *dev = state->base.dev;
13890         struct drm_i915_private *dev_priv = to_i915(dev);
13891         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
13892         struct intel_crtc *crtc;
13893         u64 put_domains[I915_MAX_PIPES] = {};
13894         intel_wakeref_t wakeref = 0;
13895         int i;
13896
13897         intel_atomic_commit_fence_wait(state);
13898
13899         drm_atomic_helper_wait_for_dependencies(&state->base);
13900
13901         if (state->modeset)
13902                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13903
13904         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13905                 if (needs_modeset(new_crtc_state) ||
13906                     new_crtc_state->update_pipe) {
13907
13908                         put_domains[crtc->pipe] =
13909                                 modeset_get_crtc_power_domains(new_crtc_state);
13910                 }
13911
13912                 if (!needs_modeset(new_crtc_state))
13913                         continue;
13914
13915                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13916
13917                 if (old_crtc_state->base.active) {
13918                         intel_crtc_disable_planes(state, crtc);
13919
13920                         /*
13921                          * We need to disable pipe CRC before disabling the pipe,
13922                          * or we race against vblank off.
13923                          */
13924                         intel_crtc_disable_pipe_crc(crtc);
13925
13926                         dev_priv->display.crtc_disable(old_crtc_state, state);
13927                         crtc->active = false;
13928                         intel_fbc_disable(crtc);
13929                         intel_disable_shared_dpll(old_crtc_state);
13930
13931                         /*
13932                          * Underruns don't always raise
13933                          * interrupts, so check manually.
13934                          */
13935                         intel_check_cpu_fifo_underruns(dev_priv);
13936                         intel_check_pch_fifo_underruns(dev_priv);
13937
13938                         /* FIXME unify this for all platforms */
13939                         if (!new_crtc_state->base.active &&
13940                             !HAS_GMCH(dev_priv) &&
13941                             dev_priv->display.initial_watermarks)
13942                                 dev_priv->display.initial_watermarks(state,
13943                                                                      new_crtc_state);
13944                 }
13945         }
13946
13947         /* FIXME: Eventually get rid of our crtc->config pointer */
13948         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13949                 crtc->config = new_crtc_state;
13950
13951         if (state->modeset) {
13952                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
13953
13954                 intel_set_cdclk_pre_plane_update(dev_priv,
13955                                                  &state->cdclk.actual,
13956                                                  &dev_priv->cdclk.actual,
13957                                                  state->cdclk.pipe);
13958
13959                 /*
13960                  * SKL workaround: bspec recommends we disable the SAGV when we
13961                  * have more then one pipe enabled
13962                  */
13963                 if (!intel_can_enable_sagv(state))
13964                         intel_disable_sagv(dev_priv);
13965
13966                 intel_modeset_verify_disabled(dev_priv, state);
13967         }
13968
13969         /* Complete the events for pipes that have now been disabled */
13970         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13971                 bool modeset = needs_modeset(new_crtc_state);
13972
13973                 /* Complete events for now disable pipes here. */
13974                 if (modeset && !new_crtc_state->base.active && new_crtc_state->base.event) {
13975                         spin_lock_irq(&dev->event_lock);
13976                         drm_crtc_send_vblank_event(&crtc->base, new_crtc_state->base.event);
13977                         spin_unlock_irq(&dev->event_lock);
13978
13979                         new_crtc_state->base.event = NULL;
13980                 }
13981         }
13982
13983         if (state->modeset)
13984                 intel_encoders_update_prepare(state);
13985
13986         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13987         dev_priv->display.update_crtcs(state);
13988
13989         if (state->modeset) {
13990                 intel_encoders_update_complete(state);
13991
13992                 intel_set_cdclk_post_plane_update(dev_priv,
13993                                                   &state->cdclk.actual,
13994                                                   &dev_priv->cdclk.actual,
13995                                                   state->cdclk.pipe);
13996         }
13997
13998         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
13999          * already, but still need the state for the delayed optimization. To
14000          * fix this:
14001          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
14002          * - schedule that vblank worker _before_ calling hw_done
14003          * - at the start of commit_tail, cancel it _synchrously
14004          * - switch over to the vblank wait helper in the core after that since
14005          *   we don't need out special handling any more.
14006          */
14007         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
14008
14009         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14010                 if (new_crtc_state->base.active &&
14011                     !needs_modeset(new_crtc_state) &&
14012                     (new_crtc_state->base.color_mgmt_changed ||
14013                      new_crtc_state->update_pipe))
14014                         intel_color_load_luts(new_crtc_state);
14015         }
14016
14017         /*
14018          * Now that the vblank has passed, we can go ahead and program the
14019          * optimal watermarks on platforms that need two-step watermark
14020          * programming.
14021          *
14022          * TODO: Move this (and other cleanup) to an async worker eventually.
14023          */
14024         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14025                 if (dev_priv->display.optimize_watermarks)
14026                         dev_priv->display.optimize_watermarks(state,
14027                                                               new_crtc_state);
14028         }
14029
14030         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14031                 intel_post_plane_update(old_crtc_state);
14032
14033                 if (put_domains[i])
14034                         modeset_put_power_domains(dev_priv, put_domains[i]);
14035
14036                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
14037         }
14038
14039         if (state->modeset)
14040                 intel_verify_planes(state);
14041
14042         if (state->modeset && intel_can_enable_sagv(state))
14043                 intel_enable_sagv(dev_priv);
14044
14045         drm_atomic_helper_commit_hw_done(&state->base);
14046
14047         if (state->modeset) {
14048                 /* As one of the primary mmio accessors, KMS has a high
14049                  * likelihood of triggering bugs in unclaimed access. After we
14050                  * finish modesetting, see if an error has been flagged, and if
14051                  * so enable debugging for the next modeset - and hope we catch
14052                  * the culprit.
14053                  */
14054                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
14055                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
14056         }
14057         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14058
14059         /*
14060          * Defer the cleanup of the old state to a separate worker to not
14061          * impede the current task (userspace for blocking modesets) that
14062          * are executed inline. For out-of-line asynchronous modesets/flips,
14063          * deferring to a new worker seems overkill, but we would place a
14064          * schedule point (cond_resched()) here anyway to keep latencies
14065          * down.
14066          */
14067         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
14068         queue_work(system_highpri_wq, &state->base.commit_work);
14069 }
14070
14071 static void intel_atomic_commit_work(struct work_struct *work)
14072 {
14073         struct intel_atomic_state *state =
14074                 container_of(work, struct intel_atomic_state, base.commit_work);
14075
14076         intel_atomic_commit_tail(state);
14077 }
14078
14079 static int __i915_sw_fence_call
14080 intel_atomic_commit_ready(struct i915_sw_fence *fence,
14081                           enum i915_sw_fence_notify notify)
14082 {
14083         struct intel_atomic_state *state =
14084                 container_of(fence, struct intel_atomic_state, commit_ready);
14085
14086         switch (notify) {
14087         case FENCE_COMPLETE:
14088                 /* we do blocking waits in the worker, nothing to do here */
14089                 break;
14090         case FENCE_FREE:
14091                 {
14092                         struct intel_atomic_helper *helper =
14093                                 &to_i915(state->base.dev)->atomic_helper;
14094
14095                         if (llist_add(&state->freed, &helper->free_list))
14096                                 schedule_work(&helper->free_work);
14097                         break;
14098                 }
14099         }
14100
14101         return NOTIFY_DONE;
14102 }
14103
14104 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
14105 {
14106         struct intel_plane_state *old_plane_state, *new_plane_state;
14107         struct intel_plane *plane;
14108         int i;
14109
14110         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
14111                                              new_plane_state, i)
14112                 i915_gem_track_fb(intel_fb_obj(old_plane_state->base.fb),
14113                                   intel_fb_obj(new_plane_state->base.fb),
14114                                   plane->frontbuffer_bit);
14115 }
14116
14117 /**
14118  * intel_atomic_commit - commit validated state object
14119  * @dev: DRM device
14120  * @state: the top-level driver state object
14121  * @nonblock: nonblocking commit
14122  *
14123  * This function commits a top-level state object that has been validated
14124  * with drm_atomic_helper_check().
14125  *
14126  * RETURNS
14127  * Zero for success or -errno.
14128  */
14129 static int intel_atomic_commit(struct drm_device *dev,
14130                                struct drm_atomic_state *state,
14131                                bool nonblock)
14132 {
14133         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14134         struct drm_i915_private *dev_priv = to_i915(dev);
14135         int ret = 0;
14136
14137         intel_state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
14138
14139         drm_atomic_state_get(state);
14140         i915_sw_fence_init(&intel_state->commit_ready,
14141                            intel_atomic_commit_ready);
14142
14143         /*
14144          * The intel_legacy_cursor_update() fast path takes care
14145          * of avoiding the vblank waits for simple cursor
14146          * movement and flips. For cursor on/off and size changes,
14147          * we want to perform the vblank waits so that watermark
14148          * updates happen during the correct frames. Gen9+ have
14149          * double buffered watermarks and so shouldn't need this.
14150          *
14151          * Unset state->legacy_cursor_update before the call to
14152          * drm_atomic_helper_setup_commit() because otherwise
14153          * drm_atomic_helper_wait_for_flip_done() is a noop and
14154          * we get FIFO underruns because we didn't wait
14155          * for vblank.
14156          *
14157          * FIXME doing watermarks and fb cleanup from a vblank worker
14158          * (assuming we had any) would solve these problems.
14159          */
14160         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
14161                 struct intel_crtc_state *new_crtc_state;
14162                 struct intel_crtc *crtc;
14163                 int i;
14164
14165                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
14166                         if (new_crtc_state->wm.need_postvbl_update ||
14167                             new_crtc_state->update_wm_post)
14168                                 state->legacy_cursor_update = false;
14169         }
14170
14171         ret = intel_atomic_prepare_commit(dev, state);
14172         if (ret) {
14173                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
14174                 i915_sw_fence_commit(&intel_state->commit_ready);
14175                 intel_runtime_pm_put(&dev_priv->runtime_pm, intel_state->wakeref);
14176                 return ret;
14177         }
14178
14179         ret = drm_atomic_helper_setup_commit(state, nonblock);
14180         if (!ret)
14181                 ret = drm_atomic_helper_swap_state(state, true);
14182
14183         if (ret) {
14184                 i915_sw_fence_commit(&intel_state->commit_ready);
14185
14186                 drm_atomic_helper_cleanup_planes(dev, state);
14187                 intel_runtime_pm_put(&dev_priv->runtime_pm, intel_state->wakeref);
14188                 return ret;
14189         }
14190         dev_priv->wm.distrust_bios_wm = false;
14191         intel_shared_dpll_swap_state(intel_state);
14192         intel_atomic_track_fbs(intel_state);
14193
14194         if (intel_state->modeset) {
14195                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
14196                        sizeof(intel_state->min_cdclk));
14197                 memcpy(dev_priv->min_voltage_level,
14198                        intel_state->min_voltage_level,
14199                        sizeof(intel_state->min_voltage_level));
14200                 dev_priv->active_crtcs = intel_state->active_crtcs;
14201                 dev_priv->cdclk.force_min_cdclk =
14202                         intel_state->cdclk.force_min_cdclk;
14203
14204                 intel_cdclk_swap_state(intel_state);
14205         }
14206
14207         drm_atomic_state_get(state);
14208         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
14209
14210         i915_sw_fence_commit(&intel_state->commit_ready);
14211         if (nonblock && intel_state->modeset) {
14212                 queue_work(dev_priv->modeset_wq, &state->commit_work);
14213         } else if (nonblock) {
14214                 queue_work(system_unbound_wq, &state->commit_work);
14215         } else {
14216                 if (intel_state->modeset)
14217                         flush_workqueue(dev_priv->modeset_wq);
14218                 intel_atomic_commit_tail(intel_state);
14219         }
14220
14221         return 0;
14222 }
14223
14224 struct wait_rps_boost {
14225         struct wait_queue_entry wait;
14226
14227         struct drm_crtc *crtc;
14228         struct i915_request *request;
14229 };
14230
14231 static int do_rps_boost(struct wait_queue_entry *_wait,
14232                         unsigned mode, int sync, void *key)
14233 {
14234         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
14235         struct i915_request *rq = wait->request;
14236
14237         /*
14238          * If we missed the vblank, but the request is already running it
14239          * is reasonable to assume that it will complete before the next
14240          * vblank without our intervention, so leave RPS alone.
14241          */
14242         if (!i915_request_started(rq))
14243                 gen6_rps_boost(rq);
14244         i915_request_put(rq);
14245
14246         drm_crtc_vblank_put(wait->crtc);
14247
14248         list_del(&wait->wait.entry);
14249         kfree(wait);
14250         return 1;
14251 }
14252
14253 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
14254                                        struct dma_fence *fence)
14255 {
14256         struct wait_rps_boost *wait;
14257
14258         if (!dma_fence_is_i915(fence))
14259                 return;
14260
14261         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
14262                 return;
14263
14264         if (drm_crtc_vblank_get(crtc))
14265                 return;
14266
14267         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
14268         if (!wait) {
14269                 drm_crtc_vblank_put(crtc);
14270                 return;
14271         }
14272
14273         wait->request = to_request(dma_fence_get(fence));
14274         wait->crtc = crtc;
14275
14276         wait->wait.func = do_rps_boost;
14277         wait->wait.flags = 0;
14278
14279         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
14280 }
14281
14282 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
14283 {
14284         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
14285         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
14286         struct drm_framebuffer *fb = plane_state->base.fb;
14287         struct i915_vma *vma;
14288
14289         if (plane->id == PLANE_CURSOR &&
14290             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
14291                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14292                 const int align = intel_cursor_alignment(dev_priv);
14293                 int err;
14294
14295                 err = i915_gem_object_attach_phys(obj, align);
14296                 if (err)
14297                         return err;
14298         }
14299
14300         vma = intel_pin_and_fence_fb_obj(fb,
14301                                          &plane_state->view,
14302                                          intel_plane_uses_fence(plane_state),
14303                                          &plane_state->flags);
14304         if (IS_ERR(vma))
14305                 return PTR_ERR(vma);
14306
14307         plane_state->vma = vma;
14308
14309         return 0;
14310 }
14311
14312 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
14313 {
14314         struct i915_vma *vma;
14315
14316         vma = fetch_and_zero(&old_plane_state->vma);
14317         if (vma)
14318                 intel_unpin_fb_vma(vma, old_plane_state->flags);
14319 }
14320
14321 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
14322 {
14323         struct i915_sched_attr attr = {
14324                 .priority = I915_PRIORITY_DISPLAY,
14325         };
14326
14327         i915_gem_object_wait_priority(obj, 0, &attr);
14328 }
14329
14330 /**
14331  * intel_prepare_plane_fb - Prepare fb for usage on plane
14332  * @plane: drm plane to prepare for
14333  * @new_state: the plane state being prepared
14334  *
14335  * Prepares a framebuffer for usage on a display plane.  Generally this
14336  * involves pinning the underlying object and updating the frontbuffer tracking
14337  * bits.  Some older platforms need special physical address handling for
14338  * cursor planes.
14339  *
14340  * Must be called with struct_mutex held.
14341  *
14342  * Returns 0 on success, negative error code on failure.
14343  */
14344 int
14345 intel_prepare_plane_fb(struct drm_plane *plane,
14346                        struct drm_plane_state *new_state)
14347 {
14348         struct intel_atomic_state *intel_state =
14349                 to_intel_atomic_state(new_state->state);
14350         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14351         struct drm_framebuffer *fb = new_state->fb;
14352         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14353         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
14354         int ret;
14355
14356         if (old_obj) {
14357                 struct intel_crtc_state *crtc_state =
14358                         intel_atomic_get_new_crtc_state(intel_state,
14359                                                         to_intel_crtc(plane->state->crtc));
14360
14361                 /* Big Hammer, we also need to ensure that any pending
14362                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
14363                  * current scanout is retired before unpinning the old
14364                  * framebuffer. Note that we rely on userspace rendering
14365                  * into the buffer attached to the pipe they are waiting
14366                  * on. If not, userspace generates a GPU hang with IPEHR
14367                  * point to the MI_WAIT_FOR_EVENT.
14368                  *
14369                  * This should only fail upon a hung GPU, in which case we
14370                  * can safely continue.
14371                  */
14372                 if (needs_modeset(crtc_state)) {
14373                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14374                                                               old_obj->base.resv, NULL,
14375                                                               false, 0,
14376                                                               GFP_KERNEL);
14377                         if (ret < 0)
14378                                 return ret;
14379                 }
14380         }
14381
14382         if (new_state->fence) { /* explicit fencing */
14383                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
14384                                                     new_state->fence,
14385                                                     I915_FENCE_TIMEOUT,
14386                                                     GFP_KERNEL);
14387                 if (ret < 0)
14388                         return ret;
14389         }
14390
14391         if (!obj)
14392                 return 0;
14393
14394         ret = i915_gem_object_pin_pages(obj);
14395         if (ret)
14396                 return ret;
14397
14398         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14399         if (ret) {
14400                 i915_gem_object_unpin_pages(obj);
14401                 return ret;
14402         }
14403
14404         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
14405
14406         mutex_unlock(&dev_priv->drm.struct_mutex);
14407         i915_gem_object_unpin_pages(obj);
14408         if (ret)
14409                 return ret;
14410
14411         fb_obj_bump_render_priority(obj);
14412         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14413
14414         if (!new_state->fence) { /* implicit fencing */
14415                 struct dma_fence *fence;
14416
14417                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14418                                                       obj->base.resv, NULL,
14419                                                       false, I915_FENCE_TIMEOUT,
14420                                                       GFP_KERNEL);
14421                 if (ret < 0)
14422                         return ret;
14423
14424                 fence = reservation_object_get_excl_rcu(obj->base.resv);
14425                 if (fence) {
14426                         add_rps_boost_after_vblank(new_state->crtc, fence);
14427                         dma_fence_put(fence);
14428                 }
14429         } else {
14430                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
14431         }
14432
14433         /*
14434          * We declare pageflips to be interactive and so merit a small bias
14435          * towards upclocking to deliver the frame on time. By only changing
14436          * the RPS thresholds to sample more regularly and aim for higher
14437          * clocks we can hopefully deliver low power workloads (like kodi)
14438          * that are not quite steady state without resorting to forcing
14439          * maximum clocks following a vblank miss (see do_rps_boost()).
14440          */
14441         if (!intel_state->rps_interactive) {
14442                 intel_rps_mark_interactive(dev_priv, true);
14443                 intel_state->rps_interactive = true;
14444         }
14445
14446         return 0;
14447 }
14448
14449 /**
14450  * intel_cleanup_plane_fb - Cleans up an fb after plane use
14451  * @plane: drm plane to clean up for
14452  * @old_state: the state from the previous modeset
14453  *
14454  * Cleans up a framebuffer that has just been removed from a plane.
14455  *
14456  * Must be called with struct_mutex held.
14457  */
14458 void
14459 intel_cleanup_plane_fb(struct drm_plane *plane,
14460                        struct drm_plane_state *old_state)
14461 {
14462         struct intel_atomic_state *intel_state =
14463                 to_intel_atomic_state(old_state->state);
14464         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14465
14466         if (intel_state->rps_interactive) {
14467                 intel_rps_mark_interactive(dev_priv, false);
14468                 intel_state->rps_interactive = false;
14469         }
14470
14471         /* Should only be called after a successful intel_prepare_plane_fb()! */
14472         mutex_lock(&dev_priv->drm.struct_mutex);
14473         intel_plane_unpin_fb(to_intel_plane_state(old_state));
14474         mutex_unlock(&dev_priv->drm.struct_mutex);
14475 }
14476
14477 int
14478 skl_max_scale(const struct intel_crtc_state *crtc_state,
14479               u32 pixel_format)
14480 {
14481         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
14482         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14483         int max_scale, mult;
14484         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
14485
14486         if (!crtc_state->base.enable)
14487                 return DRM_PLANE_HELPER_NO_SCALING;
14488
14489         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
14490         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
14491
14492         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
14493                 max_dotclk *= 2;
14494
14495         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
14496                 return DRM_PLANE_HELPER_NO_SCALING;
14497
14498         /*
14499          * skl max scale is lower of:
14500          *    close to 3 but not 3, -1 is for that purpose
14501          *            or
14502          *    cdclk/crtc_clock
14503          */
14504         mult = is_planar_yuv_format(pixel_format) ? 2 : 3;
14505         tmpclk1 = (1 << 16) * mult - 1;
14506         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
14507         max_scale = min(tmpclk1, tmpclk2);
14508
14509         return max_scale;
14510 }
14511
14512 static void intel_begin_crtc_commit(struct intel_atomic_state *state,
14513                                     struct intel_crtc *crtc)
14514 {
14515         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14516         struct intel_crtc_state *old_crtc_state =
14517                 intel_atomic_get_old_crtc_state(state, crtc);
14518         struct intel_crtc_state *new_crtc_state =
14519                 intel_atomic_get_new_crtc_state(state, crtc);
14520         bool modeset = needs_modeset(new_crtc_state);
14521
14522         /* Perform vblank evasion around commit operation */
14523         intel_pipe_update_start(new_crtc_state);
14524
14525         if (modeset)
14526                 goto out;
14527
14528         if (new_crtc_state->base.color_mgmt_changed ||
14529             new_crtc_state->update_pipe)
14530                 intel_color_commit(new_crtc_state);
14531
14532         if (new_crtc_state->update_pipe)
14533                 intel_update_pipe_config(old_crtc_state, new_crtc_state);
14534         else if (INTEL_GEN(dev_priv) >= 9)
14535                 skl_detach_scalers(new_crtc_state);
14536
14537         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14538                 bdw_set_pipemisc(new_crtc_state);
14539
14540 out:
14541         if (dev_priv->display.atomic_update_watermarks)
14542                 dev_priv->display.atomic_update_watermarks(state,
14543                                                            new_crtc_state);
14544 }
14545
14546 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14547                                   struct intel_crtc_state *crtc_state)
14548 {
14549         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14550
14551         if (!IS_GEN(dev_priv, 2))
14552                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14553
14554         if (crtc_state->has_pch_encoder) {
14555                 enum pipe pch_transcoder =
14556                         intel_crtc_pch_transcoder(crtc);
14557
14558                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14559         }
14560 }
14561
14562 static void intel_finish_crtc_commit(struct intel_atomic_state *state,
14563                                      struct intel_crtc *crtc)
14564 {
14565         struct intel_crtc_state *old_crtc_state =
14566                 intel_atomic_get_old_crtc_state(state, crtc);
14567         struct intel_crtc_state *new_crtc_state =
14568                 intel_atomic_get_new_crtc_state(state, crtc);
14569
14570         intel_pipe_update_end(new_crtc_state);
14571
14572         if (new_crtc_state->update_pipe &&
14573             !needs_modeset(new_crtc_state) &&
14574             old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED)
14575                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14576 }
14577
14578 /**
14579  * intel_plane_destroy - destroy a plane
14580  * @plane: plane to destroy
14581  *
14582  * Common destruction function for all types of planes (primary, cursor,
14583  * sprite).
14584  */
14585 void intel_plane_destroy(struct drm_plane *plane)
14586 {
14587         drm_plane_cleanup(plane);
14588         kfree(to_intel_plane(plane));
14589 }
14590
14591 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
14592                                             u32 format, u64 modifier)
14593 {
14594         switch (modifier) {
14595         case DRM_FORMAT_MOD_LINEAR:
14596         case I915_FORMAT_MOD_X_TILED:
14597                 break;
14598         default:
14599                 return false;
14600         }
14601
14602         switch (format) {
14603         case DRM_FORMAT_C8:
14604         case DRM_FORMAT_RGB565:
14605         case DRM_FORMAT_XRGB1555:
14606         case DRM_FORMAT_XRGB8888:
14607                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14608                         modifier == I915_FORMAT_MOD_X_TILED;
14609         default:
14610                 return false;
14611         }
14612 }
14613
14614 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
14615                                             u32 format, u64 modifier)
14616 {
14617         switch (modifier) {
14618         case DRM_FORMAT_MOD_LINEAR:
14619         case I915_FORMAT_MOD_X_TILED:
14620                 break;
14621         default:
14622                 return false;
14623         }
14624
14625         switch (format) {
14626         case DRM_FORMAT_C8:
14627         case DRM_FORMAT_RGB565:
14628         case DRM_FORMAT_XRGB8888:
14629         case DRM_FORMAT_XBGR8888:
14630         case DRM_FORMAT_XRGB2101010:
14631         case DRM_FORMAT_XBGR2101010:
14632                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14633                         modifier == I915_FORMAT_MOD_X_TILED;
14634         default:
14635                 return false;
14636         }
14637 }
14638
14639 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
14640                                               u32 format, u64 modifier)
14641 {
14642         return modifier == DRM_FORMAT_MOD_LINEAR &&
14643                 format == DRM_FORMAT_ARGB8888;
14644 }
14645
14646 static const struct drm_plane_funcs i965_plane_funcs = {
14647         .update_plane = drm_atomic_helper_update_plane,
14648         .disable_plane = drm_atomic_helper_disable_plane,
14649         .destroy = intel_plane_destroy,
14650         .atomic_duplicate_state = intel_plane_duplicate_state,
14651         .atomic_destroy_state = intel_plane_destroy_state,
14652         .format_mod_supported = i965_plane_format_mod_supported,
14653 };
14654
14655 static const struct drm_plane_funcs i8xx_plane_funcs = {
14656         .update_plane = drm_atomic_helper_update_plane,
14657         .disable_plane = drm_atomic_helper_disable_plane,
14658         .destroy = intel_plane_destroy,
14659         .atomic_duplicate_state = intel_plane_duplicate_state,
14660         .atomic_destroy_state = intel_plane_destroy_state,
14661         .format_mod_supported = i8xx_plane_format_mod_supported,
14662 };
14663
14664 static int
14665 intel_legacy_cursor_update(struct drm_plane *plane,
14666                            struct drm_crtc *crtc,
14667                            struct drm_framebuffer *fb,
14668                            int crtc_x, int crtc_y,
14669                            unsigned int crtc_w, unsigned int crtc_h,
14670                            u32 src_x, u32 src_y,
14671                            u32 src_w, u32 src_h,
14672                            struct drm_modeset_acquire_ctx *ctx)
14673 {
14674         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
14675         int ret;
14676         struct drm_plane_state *old_plane_state, *new_plane_state;
14677         struct intel_plane *intel_plane = to_intel_plane(plane);
14678         struct drm_framebuffer *old_fb;
14679         struct intel_crtc_state *crtc_state =
14680                 to_intel_crtc_state(crtc->state);
14681         struct intel_crtc_state *new_crtc_state;
14682
14683         /*
14684          * When crtc is inactive or there is a modeset pending,
14685          * wait for it to complete in the slowpath
14686          */
14687         if (!crtc_state->base.active || needs_modeset(crtc_state) ||
14688             crtc_state->update_pipe)
14689                 goto slow;
14690
14691         old_plane_state = plane->state;
14692         /*
14693          * Don't do an async update if there is an outstanding commit modifying
14694          * the plane.  This prevents our async update's changes from getting
14695          * overridden by a previous synchronous update's state.
14696          */
14697         if (old_plane_state->commit &&
14698             !try_wait_for_completion(&old_plane_state->commit->hw_done))
14699                 goto slow;
14700
14701         /*
14702          * If any parameters change that may affect watermarks,
14703          * take the slowpath. Only changing fb or position should be
14704          * in the fastpath.
14705          */
14706         if (old_plane_state->crtc != crtc ||
14707             old_plane_state->src_w != src_w ||
14708             old_plane_state->src_h != src_h ||
14709             old_plane_state->crtc_w != crtc_w ||
14710             old_plane_state->crtc_h != crtc_h ||
14711             !old_plane_state->fb != !fb)
14712                 goto slow;
14713
14714         new_plane_state = intel_plane_duplicate_state(plane);
14715         if (!new_plane_state)
14716                 return -ENOMEM;
14717
14718         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
14719         if (!new_crtc_state) {
14720                 ret = -ENOMEM;
14721                 goto out_free;
14722         }
14723
14724         drm_atomic_set_fb_for_plane(new_plane_state, fb);
14725
14726         new_plane_state->src_x = src_x;
14727         new_plane_state->src_y = src_y;
14728         new_plane_state->src_w = src_w;
14729         new_plane_state->src_h = src_h;
14730         new_plane_state->crtc_x = crtc_x;
14731         new_plane_state->crtc_y = crtc_y;
14732         new_plane_state->crtc_w = crtc_w;
14733         new_plane_state->crtc_h = crtc_h;
14734
14735         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
14736                                                   to_intel_plane_state(old_plane_state),
14737                                                   to_intel_plane_state(new_plane_state));
14738         if (ret)
14739                 goto out_free;
14740
14741         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14742         if (ret)
14743                 goto out_free;
14744
14745         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
14746         if (ret)
14747                 goto out_unlock;
14748
14749         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
14750
14751         old_fb = old_plane_state->fb;
14752         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
14753                           intel_plane->frontbuffer_bit);
14754
14755         /* Swap plane state */
14756         plane->state = new_plane_state;
14757
14758         /*
14759          * We cannot swap crtc_state as it may be in use by an atomic commit or
14760          * page flip that's running simultaneously. If we swap crtc_state and
14761          * destroy the old state, we will cause a use-after-free there.
14762          *
14763          * Only update active_planes, which is needed for our internal
14764          * bookkeeping. Either value will do the right thing when updating
14765          * planes atomically. If the cursor was part of the atomic update then
14766          * we would have taken the slowpath.
14767          */
14768         crtc_state->active_planes = new_crtc_state->active_planes;
14769
14770         if (plane->state->visible)
14771                 intel_update_plane(intel_plane, crtc_state,
14772                                    to_intel_plane_state(plane->state));
14773         else
14774                 intel_disable_plane(intel_plane, crtc_state);
14775
14776         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
14777
14778 out_unlock:
14779         mutex_unlock(&dev_priv->drm.struct_mutex);
14780 out_free:
14781         if (new_crtc_state)
14782                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
14783         if (ret)
14784                 intel_plane_destroy_state(plane, new_plane_state);
14785         else
14786                 intel_plane_destroy_state(plane, old_plane_state);
14787         return ret;
14788
14789 slow:
14790         return drm_atomic_helper_update_plane(plane, crtc, fb,
14791                                               crtc_x, crtc_y, crtc_w, crtc_h,
14792                                               src_x, src_y, src_w, src_h, ctx);
14793 }
14794
14795 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
14796         .update_plane = intel_legacy_cursor_update,
14797         .disable_plane = drm_atomic_helper_disable_plane,
14798         .destroy = intel_plane_destroy,
14799         .atomic_duplicate_state = intel_plane_duplicate_state,
14800         .atomic_destroy_state = intel_plane_destroy_state,
14801         .format_mod_supported = intel_cursor_format_mod_supported,
14802 };
14803
14804 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
14805                                enum i9xx_plane_id i9xx_plane)
14806 {
14807         if (!HAS_FBC(dev_priv))
14808                 return false;
14809
14810         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
14811                 return i9xx_plane == PLANE_A; /* tied to pipe A */
14812         else if (IS_IVYBRIDGE(dev_priv))
14813                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
14814                         i9xx_plane == PLANE_C;
14815         else if (INTEL_GEN(dev_priv) >= 4)
14816                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
14817         else
14818                 return i9xx_plane == PLANE_A;
14819 }
14820
14821 static struct intel_plane *
14822 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
14823 {
14824         struct intel_plane *plane;
14825         const struct drm_plane_funcs *plane_funcs;
14826         unsigned int supported_rotations;
14827         unsigned int possible_crtcs;
14828         const u64 *modifiers;
14829         const u32 *formats;
14830         int num_formats;
14831         int ret;
14832
14833         if (INTEL_GEN(dev_priv) >= 9)
14834                 return skl_universal_plane_create(dev_priv, pipe,
14835                                                   PLANE_PRIMARY);
14836
14837         plane = intel_plane_alloc();
14838         if (IS_ERR(plane))
14839                 return plane;
14840
14841         plane->pipe = pipe;
14842         /*
14843          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
14844          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
14845          */
14846         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14847                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
14848         else
14849                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
14850         plane->id = PLANE_PRIMARY;
14851         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
14852
14853         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
14854         if (plane->has_fbc) {
14855                 struct intel_fbc *fbc = &dev_priv->fbc;
14856
14857                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
14858         }
14859
14860         if (INTEL_GEN(dev_priv) >= 4) {
14861                 formats = i965_primary_formats;
14862                 num_formats = ARRAY_SIZE(i965_primary_formats);
14863                 modifiers = i9xx_format_modifiers;
14864
14865                 plane->max_stride = i9xx_plane_max_stride;
14866                 plane->update_plane = i9xx_update_plane;
14867                 plane->disable_plane = i9xx_disable_plane;
14868                 plane->get_hw_state = i9xx_plane_get_hw_state;
14869                 plane->check_plane = i9xx_plane_check;
14870
14871                 plane_funcs = &i965_plane_funcs;
14872         } else {
14873                 formats = i8xx_primary_formats;
14874                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
14875                 modifiers = i9xx_format_modifiers;
14876
14877                 plane->max_stride = i9xx_plane_max_stride;
14878                 plane->update_plane = i9xx_update_plane;
14879                 plane->disable_plane = i9xx_disable_plane;
14880                 plane->get_hw_state = i9xx_plane_get_hw_state;
14881                 plane->check_plane = i9xx_plane_check;
14882
14883                 plane_funcs = &i8xx_plane_funcs;
14884         }
14885
14886         possible_crtcs = BIT(pipe);
14887
14888         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
14889                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14890                                                possible_crtcs, plane_funcs,
14891                                                formats, num_formats, modifiers,
14892                                                DRM_PLANE_TYPE_PRIMARY,
14893                                                "primary %c", pipe_name(pipe));
14894         else
14895                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14896                                                possible_crtcs, plane_funcs,
14897                                                formats, num_formats, modifiers,
14898                                                DRM_PLANE_TYPE_PRIMARY,
14899                                                "plane %c",
14900                                                plane_name(plane->i9xx_plane));
14901         if (ret)
14902                 goto fail;
14903
14904         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
14905                 supported_rotations =
14906                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
14907                         DRM_MODE_REFLECT_X;
14908         } else if (INTEL_GEN(dev_priv) >= 4) {
14909                 supported_rotations =
14910                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
14911         } else {
14912                 supported_rotations = DRM_MODE_ROTATE_0;
14913         }
14914
14915         if (INTEL_GEN(dev_priv) >= 4)
14916                 drm_plane_create_rotation_property(&plane->base,
14917                                                    DRM_MODE_ROTATE_0,
14918                                                    supported_rotations);
14919
14920         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
14921
14922         return plane;
14923
14924 fail:
14925         intel_plane_free(plane);
14926
14927         return ERR_PTR(ret);
14928 }
14929
14930 static struct intel_plane *
14931 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
14932                           enum pipe pipe)
14933 {
14934         unsigned int possible_crtcs;
14935         struct intel_plane *cursor;
14936         int ret;
14937
14938         cursor = intel_plane_alloc();
14939         if (IS_ERR(cursor))
14940                 return cursor;
14941
14942         cursor->pipe = pipe;
14943         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
14944         cursor->id = PLANE_CURSOR;
14945         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
14946
14947         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
14948                 cursor->max_stride = i845_cursor_max_stride;
14949                 cursor->update_plane = i845_update_cursor;
14950                 cursor->disable_plane = i845_disable_cursor;
14951                 cursor->get_hw_state = i845_cursor_get_hw_state;
14952                 cursor->check_plane = i845_check_cursor;
14953         } else {
14954                 cursor->max_stride = i9xx_cursor_max_stride;
14955                 cursor->update_plane = i9xx_update_cursor;
14956                 cursor->disable_plane = i9xx_disable_cursor;
14957                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
14958                 cursor->check_plane = i9xx_check_cursor;
14959         }
14960
14961         cursor->cursor.base = ~0;
14962         cursor->cursor.cntl = ~0;
14963
14964         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14965                 cursor->cursor.size = ~0;
14966
14967         possible_crtcs = BIT(pipe);
14968
14969         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14970                                        possible_crtcs, &intel_cursor_plane_funcs,
14971                                        intel_cursor_formats,
14972                                        ARRAY_SIZE(intel_cursor_formats),
14973                                        cursor_format_modifiers,
14974                                        DRM_PLANE_TYPE_CURSOR,
14975                                        "cursor %c", pipe_name(pipe));
14976         if (ret)
14977                 goto fail;
14978
14979         if (INTEL_GEN(dev_priv) >= 4)
14980                 drm_plane_create_rotation_property(&cursor->base,
14981                                                    DRM_MODE_ROTATE_0,
14982                                                    DRM_MODE_ROTATE_0 |
14983                                                    DRM_MODE_ROTATE_180);
14984
14985         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14986
14987         return cursor;
14988
14989 fail:
14990         intel_plane_free(cursor);
14991
14992         return ERR_PTR(ret);
14993 }
14994
14995 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
14996                                     struct intel_crtc_state *crtc_state)
14997 {
14998         struct intel_crtc_scaler_state *scaler_state =
14999                 &crtc_state->scaler_state;
15000         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15001         int i;
15002
15003         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe];
15004         if (!crtc->num_scalers)
15005                 return;
15006
15007         for (i = 0; i < crtc->num_scalers; i++) {
15008                 struct intel_scaler *scaler = &scaler_state->scalers[i];
15009
15010                 scaler->in_use = 0;
15011                 scaler->mode = 0;
15012         }
15013
15014         scaler_state->scaler_id = -1;
15015 }
15016
15017 #define INTEL_CRTC_FUNCS \
15018         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15019         .set_config = drm_atomic_helper_set_config, \
15020         .destroy = intel_crtc_destroy, \
15021         .page_flip = drm_atomic_helper_page_flip, \
15022         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15023         .atomic_destroy_state = intel_crtc_destroy_state, \
15024         .set_crc_source = intel_crtc_set_crc_source, \
15025         .verify_crc_source = intel_crtc_verify_crc_source, \
15026         .get_crc_sources = intel_crtc_get_crc_sources
15027
15028 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15029         INTEL_CRTC_FUNCS,
15030
15031         .get_vblank_counter = g4x_get_vblank_counter,
15032         .enable_vblank = bdw_enable_vblank,
15033         .disable_vblank = bdw_disable_vblank,
15034 };
15035
15036 static const struct drm_crtc_funcs ilk_crtc_funcs = {
15037         INTEL_CRTC_FUNCS,
15038
15039         .get_vblank_counter = g4x_get_vblank_counter,
15040         .enable_vblank = ilk_enable_vblank,
15041         .disable_vblank = ilk_disable_vblank,
15042 };
15043
15044 static const struct drm_crtc_funcs g4x_crtc_funcs = {
15045         INTEL_CRTC_FUNCS,
15046
15047         .get_vblank_counter = g4x_get_vblank_counter,
15048         .enable_vblank = i965_enable_vblank,
15049         .disable_vblank = i965_disable_vblank,
15050 };
15051
15052 static const struct drm_crtc_funcs i965_crtc_funcs = {
15053         INTEL_CRTC_FUNCS,
15054
15055         .get_vblank_counter = i915_get_vblank_counter,
15056         .enable_vblank = i965_enable_vblank,
15057         .disable_vblank = i965_disable_vblank,
15058 };
15059
15060 static const struct drm_crtc_funcs i945gm_crtc_funcs = {
15061         INTEL_CRTC_FUNCS,
15062
15063         .get_vblank_counter = i915_get_vblank_counter,
15064         .enable_vblank = i945gm_enable_vblank,
15065         .disable_vblank = i945gm_disable_vblank,
15066 };
15067
15068 static const struct drm_crtc_funcs i915_crtc_funcs = {
15069         INTEL_CRTC_FUNCS,
15070
15071         .get_vblank_counter = i915_get_vblank_counter,
15072         .enable_vblank = i8xx_enable_vblank,
15073         .disable_vblank = i8xx_disable_vblank,
15074 };
15075
15076 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
15077         INTEL_CRTC_FUNCS,
15078
15079         /* no hw vblank counter */
15080         .enable_vblank = i8xx_enable_vblank,
15081         .disable_vblank = i8xx_disable_vblank,
15082 };
15083
15084 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15085 {
15086         const struct drm_crtc_funcs *funcs;
15087         struct intel_crtc *intel_crtc;
15088         struct intel_crtc_state *crtc_state = NULL;
15089         struct intel_plane *primary = NULL;
15090         struct intel_plane *cursor = NULL;
15091         int sprite, ret;
15092
15093         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
15094         if (!intel_crtc)
15095                 return -ENOMEM;
15096
15097         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
15098         if (!crtc_state) {
15099                 ret = -ENOMEM;
15100                 goto fail;
15101         }
15102         __drm_atomic_helper_crtc_reset(&intel_crtc->base, &crtc_state->base);
15103         intel_crtc->config = crtc_state;
15104
15105         primary = intel_primary_plane_create(dev_priv, pipe);
15106         if (IS_ERR(primary)) {
15107                 ret = PTR_ERR(primary);
15108                 goto fail;
15109         }
15110         intel_crtc->plane_ids_mask |= BIT(primary->id);
15111
15112         for_each_sprite(dev_priv, pipe, sprite) {
15113                 struct intel_plane *plane;
15114
15115                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
15116                 if (IS_ERR(plane)) {
15117                         ret = PTR_ERR(plane);
15118                         goto fail;
15119                 }
15120                 intel_crtc->plane_ids_mask |= BIT(plane->id);
15121         }
15122
15123         cursor = intel_cursor_plane_create(dev_priv, pipe);
15124         if (IS_ERR(cursor)) {
15125                 ret = PTR_ERR(cursor);
15126                 goto fail;
15127         }
15128         intel_crtc->plane_ids_mask |= BIT(cursor->id);
15129
15130         if (HAS_GMCH(dev_priv)) {
15131                 if (IS_CHERRYVIEW(dev_priv) ||
15132                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
15133                         funcs = &g4x_crtc_funcs;
15134                 else if (IS_GEN(dev_priv, 4))
15135                         funcs = &i965_crtc_funcs;
15136                 else if (IS_I945GM(dev_priv))
15137                         funcs = &i945gm_crtc_funcs;
15138                 else if (IS_GEN(dev_priv, 3))
15139                         funcs = &i915_crtc_funcs;
15140                 else
15141                         funcs = &i8xx_crtc_funcs;
15142         } else {
15143                 if (INTEL_GEN(dev_priv) >= 8)
15144                         funcs = &bdw_crtc_funcs;
15145                 else
15146                         funcs = &ilk_crtc_funcs;
15147         }
15148
15149         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
15150                                         &primary->base, &cursor->base,
15151                                         funcs, "pipe %c", pipe_name(pipe));
15152         if (ret)
15153                 goto fail;
15154
15155         intel_crtc->pipe = pipe;
15156
15157         /* initialize shared scalers */
15158         intel_crtc_init_scalers(intel_crtc, crtc_state);
15159
15160         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
15161                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
15162         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
15163
15164         if (INTEL_GEN(dev_priv) < 9) {
15165                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
15166
15167                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
15168                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
15169                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
15170         }
15171
15172         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
15173
15174         intel_color_init(intel_crtc);
15175
15176         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
15177
15178         return 0;
15179
15180 fail:
15181         /*
15182          * drm_mode_config_cleanup() will free up any
15183          * crtcs/planes already initialized.
15184          */
15185         kfree(crtc_state);
15186         kfree(intel_crtc);
15187
15188         return ret;
15189 }
15190
15191 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
15192                                       struct drm_file *file)
15193 {
15194         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
15195         struct drm_crtc *drmmode_crtc;
15196         struct intel_crtc *crtc;
15197
15198         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
15199         if (!drmmode_crtc)
15200                 return -ENOENT;
15201
15202         crtc = to_intel_crtc(drmmode_crtc);
15203         pipe_from_crtc_id->pipe = crtc->pipe;
15204
15205         return 0;
15206 }
15207
15208 static int intel_encoder_clones(struct intel_encoder *encoder)
15209 {
15210         struct drm_device *dev = encoder->base.dev;
15211         struct intel_encoder *source_encoder;
15212         int index_mask = 0;
15213         int entry = 0;
15214
15215         for_each_intel_encoder(dev, source_encoder) {
15216                 if (encoders_cloneable(encoder, source_encoder))
15217                         index_mask |= (1 << entry);
15218
15219                 entry++;
15220         }
15221
15222         return index_mask;
15223 }
15224
15225 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
15226 {
15227         if (!IS_MOBILE(dev_priv))
15228                 return false;
15229
15230         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
15231                 return false;
15232
15233         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
15234                 return false;
15235
15236         return true;
15237 }
15238
15239 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
15240 {
15241         if (INTEL_GEN(dev_priv) >= 9)
15242                 return false;
15243
15244         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
15245                 return false;
15246
15247         if (HAS_PCH_LPT_H(dev_priv) &&
15248             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
15249                 return false;
15250
15251         /* DDI E can't be used if DDI A requires 4 lanes */
15252         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
15253                 return false;
15254
15255         if (!dev_priv->vbt.int_crt_support)
15256                 return false;
15257
15258         return true;
15259 }
15260
15261 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
15262 {
15263         int pps_num;
15264         int pps_idx;
15265
15266         if (HAS_DDI(dev_priv))
15267                 return;
15268         /*
15269          * This w/a is needed at least on CPT/PPT, but to be sure apply it
15270          * everywhere where registers can be write protected.
15271          */
15272         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15273                 pps_num = 2;
15274         else
15275                 pps_num = 1;
15276
15277         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
15278                 u32 val = I915_READ(PP_CONTROL(pps_idx));
15279
15280                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
15281                 I915_WRITE(PP_CONTROL(pps_idx), val);
15282         }
15283 }
15284
15285 static void intel_pps_init(struct drm_i915_private *dev_priv)
15286 {
15287         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
15288                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
15289         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15290                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
15291         else
15292                 dev_priv->pps_mmio_base = PPS_BASE;
15293
15294         intel_pps_unlock_regs_wa(dev_priv);
15295 }
15296
15297 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
15298 {
15299         struct intel_encoder *encoder;
15300         bool dpd_is_edp = false;
15301
15302         intel_pps_init(dev_priv);
15303
15304         if (!HAS_DISPLAY(dev_priv))
15305                 return;
15306
15307         if (IS_ELKHARTLAKE(dev_priv)) {
15308                 intel_ddi_init(dev_priv, PORT_A);
15309                 intel_ddi_init(dev_priv, PORT_B);
15310                 intel_ddi_init(dev_priv, PORT_C);
15311                 intel_ddi_init(dev_priv, PORT_D);
15312                 icl_dsi_init(dev_priv);
15313         } else if (INTEL_GEN(dev_priv) >= 11) {
15314                 intel_ddi_init(dev_priv, PORT_A);
15315                 intel_ddi_init(dev_priv, PORT_B);
15316                 intel_ddi_init(dev_priv, PORT_C);
15317                 intel_ddi_init(dev_priv, PORT_D);
15318                 intel_ddi_init(dev_priv, PORT_E);
15319                 /*
15320                  * On some ICL SKUs port F is not present. No strap bits for
15321                  * this, so rely on VBT.
15322                  * Work around broken VBTs on SKUs known to have no port F.
15323                  */
15324                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
15325                     intel_bios_is_port_present(dev_priv, PORT_F))
15326                         intel_ddi_init(dev_priv, PORT_F);
15327
15328                 icl_dsi_init(dev_priv);
15329         } else if (IS_GEN9_LP(dev_priv)) {
15330                 /*
15331                  * FIXME: Broxton doesn't support port detection via the
15332                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
15333                  * detect the ports.
15334                  */
15335                 intel_ddi_init(dev_priv, PORT_A);
15336                 intel_ddi_init(dev_priv, PORT_B);
15337                 intel_ddi_init(dev_priv, PORT_C);
15338
15339                 vlv_dsi_init(dev_priv);
15340         } else if (HAS_DDI(dev_priv)) {
15341                 int found;
15342
15343                 if (intel_ddi_crt_present(dev_priv))
15344                         intel_crt_init(dev_priv);
15345
15346                 /*
15347                  * Haswell uses DDI functions to detect digital outputs.
15348                  * On SKL pre-D0 the strap isn't connected, so we assume
15349                  * it's there.
15350                  */
15351                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
15352                 /* WaIgnoreDDIAStrap: skl */
15353                 if (found || IS_GEN9_BC(dev_priv))
15354                         intel_ddi_init(dev_priv, PORT_A);
15355
15356                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
15357                  * register */
15358                 found = I915_READ(SFUSE_STRAP);
15359
15360                 if (found & SFUSE_STRAP_DDIB_DETECTED)
15361                         intel_ddi_init(dev_priv, PORT_B);
15362                 if (found & SFUSE_STRAP_DDIC_DETECTED)
15363                         intel_ddi_init(dev_priv, PORT_C);
15364                 if (found & SFUSE_STRAP_DDID_DETECTED)
15365                         intel_ddi_init(dev_priv, PORT_D);
15366                 if (found & SFUSE_STRAP_DDIF_DETECTED)
15367                         intel_ddi_init(dev_priv, PORT_F);
15368                 /*
15369                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
15370                  */
15371                 if (IS_GEN9_BC(dev_priv) &&
15372                     intel_bios_is_port_present(dev_priv, PORT_E))
15373                         intel_ddi_init(dev_priv, PORT_E);
15374
15375         } else if (HAS_PCH_SPLIT(dev_priv)) {
15376                 int found;
15377
15378                 /*
15379                  * intel_edp_init_connector() depends on this completing first,
15380                  * to prevent the registration of both eDP and LVDS and the
15381                  * incorrect sharing of the PPS.
15382                  */
15383                 intel_lvds_init(dev_priv);
15384                 intel_crt_init(dev_priv);
15385
15386                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
15387
15388                 if (ilk_has_edp_a(dev_priv))
15389                         intel_dp_init(dev_priv, DP_A, PORT_A);
15390
15391                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
15392                         /* PCH SDVOB multiplex with HDMIB */
15393                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
15394                         if (!found)
15395                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
15396                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
15397                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
15398                 }
15399
15400                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
15401                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
15402
15403                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
15404                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
15405
15406                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
15407                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
15408
15409                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
15410                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
15411         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15412                 bool has_edp, has_port;
15413
15414                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
15415                         intel_crt_init(dev_priv);
15416
15417                 /*
15418                  * The DP_DETECTED bit is the latched state of the DDC
15419                  * SDA pin at boot. However since eDP doesn't require DDC
15420                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
15421                  * eDP ports may have been muxed to an alternate function.
15422                  * Thus we can't rely on the DP_DETECTED bit alone to detect
15423                  * eDP ports. Consult the VBT as well as DP_DETECTED to
15424                  * detect eDP ports.
15425                  *
15426                  * Sadly the straps seem to be missing sometimes even for HDMI
15427                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
15428                  * and VBT for the presence of the port. Additionally we can't
15429                  * trust the port type the VBT declares as we've seen at least
15430                  * HDMI ports that the VBT claim are DP or eDP.
15431                  */
15432                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
15433                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
15434                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
15435                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
15436                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
15437                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
15438
15439                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
15440                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
15441                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
15442                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
15443                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
15444                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
15445
15446                 if (IS_CHERRYVIEW(dev_priv)) {
15447                         /*
15448                          * eDP not supported on port D,
15449                          * so no need to worry about it
15450                          */
15451                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
15452                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
15453                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
15454                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
15455                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
15456                 }
15457
15458                 vlv_dsi_init(dev_priv);
15459         } else if (IS_PINEVIEW(dev_priv)) {
15460                 intel_lvds_init(dev_priv);
15461                 intel_crt_init(dev_priv);
15462         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
15463                 bool found = false;
15464
15465                 if (IS_MOBILE(dev_priv))
15466                         intel_lvds_init(dev_priv);
15467
15468                 intel_crt_init(dev_priv);
15469
15470                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15471                         DRM_DEBUG_KMS("probing SDVOB\n");
15472                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
15473                         if (!found && IS_G4X(dev_priv)) {
15474                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
15475                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
15476                         }
15477
15478                         if (!found && IS_G4X(dev_priv))
15479                                 intel_dp_init(dev_priv, DP_B, PORT_B);
15480                 }
15481
15482                 /* Before G4X SDVOC doesn't have its own detect register */
15483
15484                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15485                         DRM_DEBUG_KMS("probing SDVOC\n");
15486                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
15487                 }
15488
15489                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
15490
15491                         if (IS_G4X(dev_priv)) {
15492                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
15493                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
15494                         }
15495                         if (IS_G4X(dev_priv))
15496                                 intel_dp_init(dev_priv, DP_C, PORT_C);
15497                 }
15498
15499                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
15500                         intel_dp_init(dev_priv, DP_D, PORT_D);
15501
15502                 if (SUPPORTS_TV(dev_priv))
15503                         intel_tv_init(dev_priv);
15504         } else if (IS_GEN(dev_priv, 2)) {
15505                 if (IS_I85X(dev_priv))
15506                         intel_lvds_init(dev_priv);
15507
15508                 intel_crt_init(dev_priv);
15509                 intel_dvo_init(dev_priv);
15510         }
15511
15512         intel_psr_init(dev_priv);
15513
15514         for_each_intel_encoder(&dev_priv->drm, encoder) {
15515                 encoder->base.possible_crtcs = encoder->crtc_mask;
15516                 encoder->base.possible_clones =
15517                         intel_encoder_clones(encoder);
15518         }
15519
15520         intel_init_pch_refclk(dev_priv);
15521
15522         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
15523 }
15524
15525 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
15526 {
15527         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
15528         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15529
15530         drm_framebuffer_cleanup(fb);
15531
15532         i915_gem_object_lock(obj);
15533         WARN_ON(!obj->framebuffer_references--);
15534         i915_gem_object_unlock(obj);
15535
15536         i915_gem_object_put(obj);
15537
15538         kfree(intel_fb);
15539 }
15540
15541 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
15542                                                 struct drm_file *file,
15543                                                 unsigned int *handle)
15544 {
15545         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15546
15547         if (obj->userptr.mm) {
15548                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
15549                 return -EINVAL;
15550         }
15551
15552         return drm_gem_handle_create(file, &obj->base, handle);
15553 }
15554
15555 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
15556                                         struct drm_file *file,
15557                                         unsigned flags, unsigned color,
15558                                         struct drm_clip_rect *clips,
15559                                         unsigned num_clips)
15560 {
15561         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15562
15563         i915_gem_object_flush_if_display(obj);
15564         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
15565
15566         return 0;
15567 }
15568
15569 static const struct drm_framebuffer_funcs intel_fb_funcs = {
15570         .destroy = intel_user_framebuffer_destroy,
15571         .create_handle = intel_user_framebuffer_create_handle,
15572         .dirty = intel_user_framebuffer_dirty,
15573 };
15574
15575 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
15576                                   struct drm_i915_gem_object *obj,
15577                                   struct drm_mode_fb_cmd2 *mode_cmd)
15578 {
15579         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
15580         struct drm_framebuffer *fb = &intel_fb->base;
15581         u32 max_stride;
15582         unsigned int tiling, stride;
15583         int ret = -EINVAL;
15584         int i;
15585
15586         i915_gem_object_lock(obj);
15587         obj->framebuffer_references++;
15588         tiling = i915_gem_object_get_tiling(obj);
15589         stride = i915_gem_object_get_stride(obj);
15590         i915_gem_object_unlock(obj);
15591
15592         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
15593                 /*
15594                  * If there's a fence, enforce that
15595                  * the fb modifier and tiling mode match.
15596                  */
15597                 if (tiling != I915_TILING_NONE &&
15598                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15599                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
15600                         goto err;
15601                 }
15602         } else {
15603                 if (tiling == I915_TILING_X) {
15604                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
15605                 } else if (tiling == I915_TILING_Y) {
15606                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
15607                         goto err;
15608                 }
15609         }
15610
15611         if (!drm_any_plane_has_format(&dev_priv->drm,
15612                                       mode_cmd->pixel_format,
15613                                       mode_cmd->modifier[0])) {
15614                 struct drm_format_name_buf format_name;
15615
15616                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
15617                               drm_get_format_name(mode_cmd->pixel_format,
15618                                                   &format_name),
15619                               mode_cmd->modifier[0]);
15620                 goto err;
15621         }
15622
15623         /*
15624          * gen2/3 display engine uses the fence if present,
15625          * so the tiling mode must match the fb modifier exactly.
15626          */
15627         if (INTEL_GEN(dev_priv) < 4 &&
15628             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15629                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
15630                 goto err;
15631         }
15632
15633         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
15634                                          mode_cmd->modifier[0]);
15635         if (mode_cmd->pitches[0] > max_stride) {
15636                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
15637                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
15638                               "tiled" : "linear",
15639                               mode_cmd->pitches[0], max_stride);
15640                 goto err;
15641         }
15642
15643         /*
15644          * If there's a fence, enforce that
15645          * the fb pitch and fence stride match.
15646          */
15647         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
15648                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
15649                               mode_cmd->pitches[0], stride);
15650                 goto err;
15651         }
15652
15653         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
15654         if (mode_cmd->offsets[0] != 0)
15655                 goto err;
15656
15657         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
15658
15659         for (i = 0; i < fb->format->num_planes; i++) {
15660                 u32 stride_alignment;
15661
15662                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
15663                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
15664                         goto err;
15665                 }
15666
15667                 stride_alignment = intel_fb_stride_alignment(fb, i);
15668
15669                 /*
15670                  * Display WA #0531: skl,bxt,kbl,glk
15671                  *
15672                  * Render decompression and plane width > 3840
15673                  * combined with horizontal panning requires the
15674                  * plane stride to be a multiple of 4. We'll just
15675                  * require the entire fb to accommodate that to avoid
15676                  * potential runtime errors at plane configuration time.
15677                  */
15678                 if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 &&
15679                     is_ccs_modifier(fb->modifier))
15680                         stride_alignment *= 4;
15681
15682                 if (fb->pitches[i] & (stride_alignment - 1)) {
15683                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
15684                                       i, fb->pitches[i], stride_alignment);
15685                         goto err;
15686                 }
15687
15688                 fb->obj[i] = &obj->base;
15689         }
15690
15691         ret = intel_fill_fb_info(dev_priv, fb);
15692         if (ret)
15693                 goto err;
15694
15695         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
15696         if (ret) {
15697                 DRM_ERROR("framebuffer init failed %d\n", ret);
15698                 goto err;
15699         }
15700
15701         return 0;
15702
15703 err:
15704         i915_gem_object_lock(obj);
15705         obj->framebuffer_references--;
15706         i915_gem_object_unlock(obj);
15707         return ret;
15708 }
15709
15710 static struct drm_framebuffer *
15711 intel_user_framebuffer_create(struct drm_device *dev,
15712                               struct drm_file *filp,
15713                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
15714 {
15715         struct drm_framebuffer *fb;
15716         struct drm_i915_gem_object *obj;
15717         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
15718
15719         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
15720         if (!obj)
15721                 return ERR_PTR(-ENOENT);
15722
15723         fb = intel_framebuffer_create(obj, &mode_cmd);
15724         if (IS_ERR(fb))
15725                 i915_gem_object_put(obj);
15726
15727         return fb;
15728 }
15729
15730 static void intel_atomic_state_free(struct drm_atomic_state *state)
15731 {
15732         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
15733
15734         drm_atomic_state_default_release(state);
15735
15736         i915_sw_fence_fini(&intel_state->commit_ready);
15737
15738         kfree(state);
15739 }
15740
15741 static enum drm_mode_status
15742 intel_mode_valid(struct drm_device *dev,
15743                  const struct drm_display_mode *mode)
15744 {
15745         struct drm_i915_private *dev_priv = to_i915(dev);
15746         int hdisplay_max, htotal_max;
15747         int vdisplay_max, vtotal_max;
15748
15749         /*
15750          * Can't reject DBLSCAN here because Xorg ddxen can add piles
15751          * of DBLSCAN modes to the output's mode list when they detect
15752          * the scaling mode property on the connector. And they don't
15753          * ask the kernel to validate those modes in any way until
15754          * modeset time at which point the client gets a protocol error.
15755          * So in order to not upset those clients we silently ignore the
15756          * DBLSCAN flag on such connectors. For other connectors we will
15757          * reject modes with the DBLSCAN flag in encoder->compute_config().
15758          * And we always reject DBLSCAN modes in connector->mode_valid()
15759          * as we never want such modes on the connector's mode list.
15760          */
15761
15762         if (mode->vscan > 1)
15763                 return MODE_NO_VSCAN;
15764
15765         if (mode->flags & DRM_MODE_FLAG_HSKEW)
15766                 return MODE_H_ILLEGAL;
15767
15768         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
15769                            DRM_MODE_FLAG_NCSYNC |
15770                            DRM_MODE_FLAG_PCSYNC))
15771                 return MODE_HSYNC;
15772
15773         if (mode->flags & (DRM_MODE_FLAG_BCAST |
15774                            DRM_MODE_FLAG_PIXMUX |
15775                            DRM_MODE_FLAG_CLKDIV2))
15776                 return MODE_BAD;
15777
15778         if (INTEL_GEN(dev_priv) >= 9 ||
15779             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
15780                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
15781                 vdisplay_max = 4096;
15782                 htotal_max = 8192;
15783                 vtotal_max = 8192;
15784         } else if (INTEL_GEN(dev_priv) >= 3) {
15785                 hdisplay_max = 4096;
15786                 vdisplay_max = 4096;
15787                 htotal_max = 8192;
15788                 vtotal_max = 8192;
15789         } else {
15790                 hdisplay_max = 2048;
15791                 vdisplay_max = 2048;
15792                 htotal_max = 4096;
15793                 vtotal_max = 4096;
15794         }
15795
15796         if (mode->hdisplay > hdisplay_max ||
15797             mode->hsync_start > htotal_max ||
15798             mode->hsync_end > htotal_max ||
15799             mode->htotal > htotal_max)
15800                 return MODE_H_ILLEGAL;
15801
15802         if (mode->vdisplay > vdisplay_max ||
15803             mode->vsync_start > vtotal_max ||
15804             mode->vsync_end > vtotal_max ||
15805             mode->vtotal > vtotal_max)
15806                 return MODE_V_ILLEGAL;
15807
15808         return MODE_OK;
15809 }
15810
15811 static const struct drm_mode_config_funcs intel_mode_funcs = {
15812         .fb_create = intel_user_framebuffer_create,
15813         .get_format_info = intel_get_format_info,
15814         .output_poll_changed = intel_fbdev_output_poll_changed,
15815         .mode_valid = intel_mode_valid,
15816         .atomic_check = intel_atomic_check,
15817         .atomic_commit = intel_atomic_commit,
15818         .atomic_state_alloc = intel_atomic_state_alloc,
15819         .atomic_state_clear = intel_atomic_state_clear,
15820         .atomic_state_free = intel_atomic_state_free,
15821 };
15822
15823 /**
15824  * intel_init_display_hooks - initialize the display modesetting hooks
15825  * @dev_priv: device private
15826  */
15827 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
15828 {
15829         intel_init_cdclk_hooks(dev_priv);
15830
15831         if (INTEL_GEN(dev_priv) >= 9) {
15832                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15833                 dev_priv->display.get_initial_plane_config =
15834                         skylake_get_initial_plane_config;
15835                 dev_priv->display.crtc_compute_clock =
15836                         haswell_crtc_compute_clock;
15837                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15838                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15839         } else if (HAS_DDI(dev_priv)) {
15840                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15841                 dev_priv->display.get_initial_plane_config =
15842                         i9xx_get_initial_plane_config;
15843                 dev_priv->display.crtc_compute_clock =
15844                         haswell_crtc_compute_clock;
15845                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15846                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15847         } else if (HAS_PCH_SPLIT(dev_priv)) {
15848                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
15849                 dev_priv->display.get_initial_plane_config =
15850                         i9xx_get_initial_plane_config;
15851                 dev_priv->display.crtc_compute_clock =
15852                         ironlake_crtc_compute_clock;
15853                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
15854                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
15855         } else if (IS_CHERRYVIEW(dev_priv)) {
15856                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15857                 dev_priv->display.get_initial_plane_config =
15858                         i9xx_get_initial_plane_config;
15859                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
15860                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15861                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15862         } else if (IS_VALLEYVIEW(dev_priv)) {
15863                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15864                 dev_priv->display.get_initial_plane_config =
15865                         i9xx_get_initial_plane_config;
15866                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
15867                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15868                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15869         } else if (IS_G4X(dev_priv)) {
15870                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15871                 dev_priv->display.get_initial_plane_config =
15872                         i9xx_get_initial_plane_config;
15873                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
15874                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15875                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15876         } else if (IS_PINEVIEW(dev_priv)) {
15877                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15878                 dev_priv->display.get_initial_plane_config =
15879                         i9xx_get_initial_plane_config;
15880                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
15881                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15882                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15883         } else if (!IS_GEN(dev_priv, 2)) {
15884                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15885                 dev_priv->display.get_initial_plane_config =
15886                         i9xx_get_initial_plane_config;
15887                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
15888                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15889                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15890         } else {
15891                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15892                 dev_priv->display.get_initial_plane_config =
15893                         i9xx_get_initial_plane_config;
15894                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
15895                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15896                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15897         }
15898
15899         if (IS_GEN(dev_priv, 5)) {
15900                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
15901         } else if (IS_GEN(dev_priv, 6)) {
15902                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
15903         } else if (IS_IVYBRIDGE(dev_priv)) {
15904                 /* FIXME: detect B0+ stepping and use auto training */
15905                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
15906         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
15907                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
15908         }
15909
15910         if (INTEL_GEN(dev_priv) >= 9)
15911                 dev_priv->display.update_crtcs = skl_update_crtcs;
15912         else
15913                 dev_priv->display.update_crtcs = intel_update_crtcs;
15914 }
15915
15916 static i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
15917 {
15918         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15919                 return VLV_VGACNTRL;
15920         else if (INTEL_GEN(dev_priv) >= 5)
15921                 return CPU_VGACNTRL;
15922         else
15923                 return VGACNTRL;
15924 }
15925
15926 /* Disable the VGA plane that we never use */
15927 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15928 {
15929         struct pci_dev *pdev = dev_priv->drm.pdev;
15930         u8 sr1;
15931         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15932
15933         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15934         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15935         outb(SR01, VGA_SR_INDEX);
15936         sr1 = inb(VGA_SR_DATA);
15937         outb(sr1 | 1<<5, VGA_SR_DATA);
15938         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15939         udelay(300);
15940
15941         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15942         POSTING_READ(vga_reg);
15943 }
15944
15945 void intel_modeset_init_hw(struct drm_device *dev)
15946 {
15947         struct drm_i915_private *dev_priv = to_i915(dev);
15948
15949         intel_update_cdclk(dev_priv);
15950         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15951         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15952 }
15953
15954 /*
15955  * Calculate what we think the watermarks should be for the state we've read
15956  * out of the hardware and then immediately program those watermarks so that
15957  * we ensure the hardware settings match our internal state.
15958  *
15959  * We can calculate what we think WM's should be by creating a duplicate of the
15960  * current state (which was constructed during hardware readout) and running it
15961  * through the atomic check code to calculate new watermark values in the
15962  * state object.
15963  */
15964 static void sanitize_watermarks(struct drm_device *dev)
15965 {
15966         struct drm_i915_private *dev_priv = to_i915(dev);
15967         struct drm_atomic_state *state;
15968         struct intel_atomic_state *intel_state;
15969         struct intel_crtc *crtc;
15970         struct intel_crtc_state *crtc_state;
15971         struct drm_modeset_acquire_ctx ctx;
15972         int ret;
15973         int i;
15974
15975         /* Only supported on platforms that use atomic watermark design */
15976         if (!dev_priv->display.optimize_watermarks)
15977                 return;
15978
15979         /*
15980          * We need to hold connection_mutex before calling duplicate_state so
15981          * that the connector loop is protected.
15982          */
15983         drm_modeset_acquire_init(&ctx, 0);
15984 retry:
15985         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15986         if (ret == -EDEADLK) {
15987                 drm_modeset_backoff(&ctx);
15988                 goto retry;
15989         } else if (WARN_ON(ret)) {
15990                 goto fail;
15991         }
15992
15993         state = drm_atomic_helper_duplicate_state(dev, &ctx);
15994         if (WARN_ON(IS_ERR(state)))
15995                 goto fail;
15996
15997         intel_state = to_intel_atomic_state(state);
15998
15999         /*
16000          * Hardware readout is the only time we don't want to calculate
16001          * intermediate watermarks (since we don't trust the current
16002          * watermarks).
16003          */
16004         if (!HAS_GMCH(dev_priv))
16005                 intel_state->skip_intermediate_wm = true;
16006
16007         ret = intel_atomic_check(dev, state);
16008         if (ret) {
16009                 /*
16010                  * If we fail here, it means that the hardware appears to be
16011                  * programmed in a way that shouldn't be possible, given our
16012                  * understanding of watermark requirements.  This might mean a
16013                  * mistake in the hardware readout code or a mistake in the
16014                  * watermark calculations for a given platform.  Raise a WARN
16015                  * so that this is noticeable.
16016                  *
16017                  * If this actually happens, we'll have to just leave the
16018                  * BIOS-programmed watermarks untouched and hope for the best.
16019                  */
16020                 WARN(true, "Could not determine valid watermarks for inherited state\n");
16021                 goto put_state;
16022         }
16023
16024         /* Write calculated watermark values back */
16025         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
16026                 crtc_state->wm.need_postvbl_update = true;
16027                 dev_priv->display.optimize_watermarks(intel_state, crtc_state);
16028
16029                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
16030         }
16031
16032 put_state:
16033         drm_atomic_state_put(state);
16034 fail:
16035         drm_modeset_drop_locks(&ctx);
16036         drm_modeset_acquire_fini(&ctx);
16037 }
16038
16039 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
16040 {
16041         if (IS_GEN(dev_priv, 5)) {
16042                 u32 fdi_pll_clk =
16043                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
16044
16045                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
16046         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
16047                 dev_priv->fdi_pll_freq = 270000;
16048         } else {
16049                 return;
16050         }
16051
16052         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
16053 }
16054
16055 static int intel_initial_commit(struct drm_device *dev)
16056 {
16057         struct drm_atomic_state *state = NULL;
16058         struct drm_modeset_acquire_ctx ctx;
16059         struct drm_crtc *crtc;
16060         struct drm_crtc_state *crtc_state;
16061         int ret = 0;
16062
16063         state = drm_atomic_state_alloc(dev);
16064         if (!state)
16065                 return -ENOMEM;
16066
16067         drm_modeset_acquire_init(&ctx, 0);
16068
16069 retry:
16070         state->acquire_ctx = &ctx;
16071
16072         drm_for_each_crtc(crtc, dev) {
16073                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
16074                 if (IS_ERR(crtc_state)) {
16075                         ret = PTR_ERR(crtc_state);
16076                         goto out;
16077                 }
16078
16079                 if (crtc_state->active) {
16080                         ret = drm_atomic_add_affected_planes(state, crtc);
16081                         if (ret)
16082                                 goto out;
16083
16084                         /*
16085                          * FIXME hack to force a LUT update to avoid the
16086                          * plane update forcing the pipe gamma on without
16087                          * having a proper LUT loaded. Remove once we
16088                          * have readout for pipe gamma enable.
16089                          */
16090                         crtc_state->color_mgmt_changed = true;
16091                 }
16092         }
16093
16094         ret = drm_atomic_commit(state);
16095
16096 out:
16097         if (ret == -EDEADLK) {
16098                 drm_atomic_state_clear(state);
16099                 drm_modeset_backoff(&ctx);
16100                 goto retry;
16101         }
16102
16103         drm_atomic_state_put(state);
16104
16105         drm_modeset_drop_locks(&ctx);
16106         drm_modeset_acquire_fini(&ctx);
16107
16108         return ret;
16109 }
16110
16111 int intel_modeset_init(struct drm_device *dev)
16112 {
16113         struct drm_i915_private *dev_priv = to_i915(dev);
16114         struct i915_ggtt *ggtt = &dev_priv->ggtt;
16115         enum pipe pipe;
16116         struct intel_crtc *crtc;
16117         int ret;
16118
16119         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
16120
16121         drm_mode_config_init(dev);
16122
16123         ret = intel_bw_init(dev_priv);
16124         if (ret)
16125                 return ret;
16126
16127         dev->mode_config.min_width = 0;
16128         dev->mode_config.min_height = 0;
16129
16130         dev->mode_config.preferred_depth = 24;
16131         dev->mode_config.prefer_shadow = 1;
16132
16133         dev->mode_config.allow_fb_modifiers = true;
16134
16135         dev->mode_config.funcs = &intel_mode_funcs;
16136
16137         init_llist_head(&dev_priv->atomic_helper.free_list);
16138         INIT_WORK(&dev_priv->atomic_helper.free_work,
16139                   intel_atomic_helper_free_state_worker);
16140
16141         intel_init_quirks(dev_priv);
16142
16143         intel_fbc_init(dev_priv);
16144
16145         intel_init_pm(dev_priv);
16146
16147         /*
16148          * There may be no VBT; and if the BIOS enabled SSC we can
16149          * just keep using it to avoid unnecessary flicker.  Whereas if the
16150          * BIOS isn't using it, don't assume it will work even if the VBT
16151          * indicates as much.
16152          */
16153         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
16154                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
16155                                             DREF_SSC1_ENABLE);
16156
16157                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
16158                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
16159                                      bios_lvds_use_ssc ? "en" : "dis",
16160                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
16161                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
16162                 }
16163         }
16164
16165         /*
16166          * Maximum framebuffer dimensions, chosen to match
16167          * the maximum render engine surface size on gen4+.
16168          */
16169         if (INTEL_GEN(dev_priv) >= 7) {
16170                 dev->mode_config.max_width = 16384;
16171                 dev->mode_config.max_height = 16384;
16172         } else if (INTEL_GEN(dev_priv) >= 4) {
16173                 dev->mode_config.max_width = 8192;
16174                 dev->mode_config.max_height = 8192;
16175         } else if (IS_GEN(dev_priv, 3)) {
16176                 dev->mode_config.max_width = 4096;
16177                 dev->mode_config.max_height = 4096;
16178         } else {
16179                 dev->mode_config.max_width = 2048;
16180                 dev->mode_config.max_height = 2048;
16181         }
16182
16183         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16184                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
16185                 dev->mode_config.cursor_height = 1023;
16186         } else if (IS_GEN(dev_priv, 2)) {
16187                 dev->mode_config.cursor_width = 64;
16188                 dev->mode_config.cursor_height = 64;
16189         } else {
16190                 dev->mode_config.cursor_width = 256;
16191                 dev->mode_config.cursor_height = 256;
16192         }
16193
16194         dev->mode_config.fb_base = ggtt->gmadr.start;
16195
16196         DRM_DEBUG_KMS("%d display pipe%s available.\n",
16197                       INTEL_INFO(dev_priv)->num_pipes,
16198                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
16199
16200         for_each_pipe(dev_priv, pipe) {
16201                 ret = intel_crtc_init(dev_priv, pipe);
16202                 if (ret) {
16203                         drm_mode_config_cleanup(dev);
16204                         return ret;
16205                 }
16206         }
16207
16208         intel_shared_dpll_init(dev);
16209         intel_update_fdi_pll_freq(dev_priv);
16210
16211         intel_update_czclk(dev_priv);
16212         intel_modeset_init_hw(dev);
16213
16214         intel_hdcp_component_init(dev_priv);
16215
16216         if (dev_priv->max_cdclk_freq == 0)
16217                 intel_update_max_cdclk(dev_priv);
16218
16219         /* Just disable it once at startup */
16220         i915_disable_vga(dev_priv);
16221         intel_setup_outputs(dev_priv);
16222
16223         drm_modeset_lock_all(dev);
16224         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
16225         drm_modeset_unlock_all(dev);
16226
16227         for_each_intel_crtc(dev, crtc) {
16228                 struct intel_initial_plane_config plane_config = {};
16229
16230                 if (!crtc->active)
16231                         continue;
16232
16233                 /*
16234                  * Note that reserving the BIOS fb up front prevents us
16235                  * from stuffing other stolen allocations like the ring
16236                  * on top.  This prevents some ugliness at boot time, and
16237                  * can even allow for smooth boot transitions if the BIOS
16238                  * fb is large enough for the active pipe configuration.
16239                  */
16240                 dev_priv->display.get_initial_plane_config(crtc,
16241                                                            &plane_config);
16242
16243                 /*
16244                  * If the fb is shared between multiple heads, we'll
16245                  * just get the first one.
16246                  */
16247                 intel_find_initial_plane_obj(crtc, &plane_config);
16248         }
16249
16250         /*
16251          * Make sure hardware watermarks really match the state we read out.
16252          * Note that we need to do this after reconstructing the BIOS fb's
16253          * since the watermark calculation done here will use pstate->fb.
16254          */
16255         if (!HAS_GMCH(dev_priv))
16256                 sanitize_watermarks(dev);
16257
16258         /*
16259          * Force all active planes to recompute their states. So that on
16260          * mode_setcrtc after probe, all the intel_plane_state variables
16261          * are already calculated and there is no assert_plane warnings
16262          * during bootup.
16263          */
16264         ret = intel_initial_commit(dev);
16265         if (ret)
16266                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
16267
16268         return 0;
16269 }
16270
16271 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16272 {
16273         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16274         /* 640x480@60Hz, ~25175 kHz */
16275         struct dpll clock = {
16276                 .m1 = 18,
16277                 .m2 = 7,
16278                 .p1 = 13,
16279                 .p2 = 4,
16280                 .n = 2,
16281         };
16282         u32 dpll, fp;
16283         int i;
16284
16285         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
16286
16287         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
16288                       pipe_name(pipe), clock.vco, clock.dot);
16289
16290         fp = i9xx_dpll_compute_fp(&clock);
16291         dpll = DPLL_DVO_2X_MODE |
16292                 DPLL_VGA_MODE_DIS |
16293                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
16294                 PLL_P2_DIVIDE_BY_4 |
16295                 PLL_REF_INPUT_DREFCLK |
16296                 DPLL_VCO_ENABLE;
16297
16298         I915_WRITE(FP0(pipe), fp);
16299         I915_WRITE(FP1(pipe), fp);
16300
16301         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
16302         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
16303         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
16304         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
16305         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
16306         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
16307         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
16308
16309         /*
16310          * Apparently we need to have VGA mode enabled prior to changing
16311          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
16312          * dividers, even though the register value does change.
16313          */
16314         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
16315         I915_WRITE(DPLL(pipe), dpll);
16316
16317         /* Wait for the clocks to stabilize. */
16318         POSTING_READ(DPLL(pipe));
16319         udelay(150);
16320
16321         /* The pixel multiplier can only be updated once the
16322          * DPLL is enabled and the clocks are stable.
16323          *
16324          * So write it again.
16325          */
16326         I915_WRITE(DPLL(pipe), dpll);
16327
16328         /* We do this three times for luck */
16329         for (i = 0; i < 3 ; i++) {
16330                 I915_WRITE(DPLL(pipe), dpll);
16331                 POSTING_READ(DPLL(pipe));
16332                 udelay(150); /* wait for warmup */
16333         }
16334
16335         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
16336         POSTING_READ(PIPECONF(pipe));
16337
16338         intel_wait_for_pipe_scanline_moving(crtc);
16339 }
16340
16341 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16342 {
16343         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16344
16345         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
16346                       pipe_name(pipe));
16347
16348         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
16349         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
16350         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
16351         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
16352         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
16353
16354         I915_WRITE(PIPECONF(pipe), 0);
16355         POSTING_READ(PIPECONF(pipe));
16356
16357         intel_wait_for_pipe_scanline_stopped(crtc);
16358
16359         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
16360         POSTING_READ(DPLL(pipe));
16361 }
16362
16363 static void
16364 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
16365 {
16366         struct intel_crtc *crtc;
16367
16368         if (INTEL_GEN(dev_priv) >= 4)
16369                 return;
16370
16371         for_each_intel_crtc(&dev_priv->drm, crtc) {
16372                 struct intel_plane *plane =
16373                         to_intel_plane(crtc->base.primary);
16374                 struct intel_crtc *plane_crtc;
16375                 enum pipe pipe;
16376
16377                 if (!plane->get_hw_state(plane, &pipe))
16378                         continue;
16379
16380                 if (pipe == crtc->pipe)
16381                         continue;
16382
16383                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
16384                               plane->base.base.id, plane->base.name);
16385
16386                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16387                 intel_plane_disable_noatomic(plane_crtc, plane);
16388         }
16389 }
16390
16391 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
16392 {
16393         struct drm_device *dev = crtc->base.dev;
16394         struct intel_encoder *encoder;
16395
16396         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
16397                 return true;
16398
16399         return false;
16400 }
16401
16402 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
16403 {
16404         struct drm_device *dev = encoder->base.dev;
16405         struct intel_connector *connector;
16406
16407         for_each_connector_on_encoder(dev, &encoder->base, connector)
16408                 return connector;
16409
16410         return NULL;
16411 }
16412
16413 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
16414                               enum pipe pch_transcoder)
16415 {
16416         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
16417                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
16418 }
16419
16420 static void intel_sanitize_crtc(struct intel_crtc *crtc,
16421                                 struct drm_modeset_acquire_ctx *ctx)
16422 {
16423         struct drm_device *dev = crtc->base.dev;
16424         struct drm_i915_private *dev_priv = to_i915(dev);
16425         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
16426         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
16427
16428         /* Clear any frame start delays used for debugging left by the BIOS */
16429         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
16430                 i915_reg_t reg = PIPECONF(cpu_transcoder);
16431
16432                 I915_WRITE(reg,
16433                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
16434         }
16435
16436         if (crtc_state->base.active) {
16437                 struct intel_plane *plane;
16438
16439                 /* Disable everything but the primary plane */
16440                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
16441                         const struct intel_plane_state *plane_state =
16442                                 to_intel_plane_state(plane->base.state);
16443
16444                         if (plane_state->base.visible &&
16445                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
16446                                 intel_plane_disable_noatomic(crtc, plane);
16447                 }
16448
16449                 /*
16450                  * Disable any background color set by the BIOS, but enable the
16451                  * gamma and CSC to match how we program our planes.
16452                  */
16453                 if (INTEL_GEN(dev_priv) >= 9)
16454                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
16455                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
16456                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
16457         }
16458
16459         /* Adjust the state of the output pipe according to whether we
16460          * have active connectors/encoders. */
16461         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
16462                 intel_crtc_disable_noatomic(&crtc->base, ctx);
16463
16464         if (crtc_state->base.active || HAS_GMCH(dev_priv)) {
16465                 /*
16466                  * We start out with underrun reporting disabled to avoid races.
16467                  * For correct bookkeeping mark this on active crtcs.
16468                  *
16469                  * Also on gmch platforms we dont have any hardware bits to
16470                  * disable the underrun reporting. Which means we need to start
16471                  * out with underrun reporting disabled also on inactive pipes,
16472                  * since otherwise we'll complain about the garbage we read when
16473                  * e.g. coming up after runtime pm.
16474                  *
16475                  * No protection against concurrent access is required - at
16476                  * worst a fifo underrun happens which also sets this to false.
16477                  */
16478                 crtc->cpu_fifo_underrun_disabled = true;
16479                 /*
16480                  * We track the PCH trancoder underrun reporting state
16481                  * within the crtc. With crtc for pipe A housing the underrun
16482                  * reporting state for PCH transcoder A, crtc for pipe B housing
16483                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
16484                  * and marking underrun reporting as disabled for the non-existing
16485                  * PCH transcoders B and C would prevent enabling the south
16486                  * error interrupt (see cpt_can_enable_serr_int()).
16487                  */
16488                 if (has_pch_trancoder(dev_priv, crtc->pipe))
16489                         crtc->pch_fifo_underrun_disabled = true;
16490         }
16491 }
16492
16493 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
16494 {
16495         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
16496
16497         /*
16498          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
16499          * the hardware when a high res displays plugged in. DPLL P
16500          * divider is zero, and the pipe timings are bonkers. We'll
16501          * try to disable everything in that case.
16502          *
16503          * FIXME would be nice to be able to sanitize this state
16504          * without several WARNs, but for now let's take the easy
16505          * road.
16506          */
16507         return IS_GEN(dev_priv, 6) &&
16508                 crtc_state->base.active &&
16509                 crtc_state->shared_dpll &&
16510                 crtc_state->port_clock == 0;
16511 }
16512
16513 static void intel_sanitize_encoder(struct intel_encoder *encoder)
16514 {
16515         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
16516         struct intel_connector *connector;
16517         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
16518         struct intel_crtc_state *crtc_state = crtc ?
16519                 to_intel_crtc_state(crtc->base.state) : NULL;
16520
16521         /* We need to check both for a crtc link (meaning that the
16522          * encoder is active and trying to read from a pipe) and the
16523          * pipe itself being active. */
16524         bool has_active_crtc = crtc_state &&
16525                 crtc_state->base.active;
16526
16527         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
16528                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
16529                               pipe_name(crtc->pipe));
16530                 has_active_crtc = false;
16531         }
16532
16533         connector = intel_encoder_find_connector(encoder);
16534         if (connector && !has_active_crtc) {
16535                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
16536                               encoder->base.base.id,
16537                               encoder->base.name);
16538
16539                 /* Connector is active, but has no active pipe. This is
16540                  * fallout from our resume register restoring. Disable
16541                  * the encoder manually again. */
16542                 if (crtc_state) {
16543                         struct drm_encoder *best_encoder;
16544
16545                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
16546                                       encoder->base.base.id,
16547                                       encoder->base.name);
16548
16549                         /* avoid oopsing in case the hooks consult best_encoder */
16550                         best_encoder = connector->base.state->best_encoder;
16551                         connector->base.state->best_encoder = &encoder->base;
16552
16553                         if (encoder->disable)
16554                                 encoder->disable(encoder, crtc_state,
16555                                                  connector->base.state);
16556                         if (encoder->post_disable)
16557                                 encoder->post_disable(encoder, crtc_state,
16558                                                       connector->base.state);
16559
16560                         connector->base.state->best_encoder = best_encoder;
16561                 }
16562                 encoder->base.crtc = NULL;
16563
16564                 /* Inconsistent output/port/pipe state happens presumably due to
16565                  * a bug in one of the get_hw_state functions. Or someplace else
16566                  * in our code, like the register restore mess on resume. Clamp
16567                  * things to off as a safer default. */
16568
16569                 connector->base.dpms = DRM_MODE_DPMS_OFF;
16570                 connector->base.encoder = NULL;
16571         }
16572
16573         /* notify opregion of the sanitized encoder state */
16574         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
16575
16576         if (INTEL_GEN(dev_priv) >= 11)
16577                 icl_sanitize_encoder_pll_mapping(encoder);
16578 }
16579
16580 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
16581 {
16582         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
16583
16584         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
16585                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
16586                 i915_disable_vga(dev_priv);
16587         }
16588 }
16589
16590 void i915_redisable_vga(struct drm_i915_private *dev_priv)
16591 {
16592         intel_wakeref_t wakeref;
16593
16594         /*
16595          * This function can be called both from intel_modeset_setup_hw_state or
16596          * at a very early point in our resume sequence, where the power well
16597          * structures are not yet restored. Since this function is at a very
16598          * paranoid "someone might have enabled VGA while we were not looking"
16599          * level, just check if the power well is enabled instead of trying to
16600          * follow the "don't touch the power well if we don't need it" policy
16601          * the rest of the driver uses.
16602          */
16603         wakeref = intel_display_power_get_if_enabled(dev_priv,
16604                                                      POWER_DOMAIN_VGA);
16605         if (!wakeref)
16606                 return;
16607
16608         i915_redisable_vga_power_on(dev_priv);
16609
16610         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref);
16611 }
16612
16613 /* FIXME read out full plane state for all planes */
16614 static void readout_plane_state(struct drm_i915_private *dev_priv)
16615 {
16616         struct intel_plane *plane;
16617         struct intel_crtc *crtc;
16618
16619         for_each_intel_plane(&dev_priv->drm, plane) {
16620                 struct intel_plane_state *plane_state =
16621                         to_intel_plane_state(plane->base.state);
16622                 struct intel_crtc_state *crtc_state;
16623                 enum pipe pipe = PIPE_A;
16624                 bool visible;
16625
16626                 visible = plane->get_hw_state(plane, &pipe);
16627
16628                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16629                 crtc_state = to_intel_crtc_state(crtc->base.state);
16630
16631                 intel_set_plane_visible(crtc_state, plane_state, visible);
16632
16633                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
16634                               plane->base.base.id, plane->base.name,
16635                               enableddisabled(visible), pipe_name(pipe));
16636         }
16637
16638         for_each_intel_crtc(&dev_priv->drm, crtc) {
16639                 struct intel_crtc_state *crtc_state =
16640                         to_intel_crtc_state(crtc->base.state);
16641
16642                 fixup_active_planes(crtc_state);
16643         }
16644 }
16645
16646 static void intel_modeset_readout_hw_state(struct drm_device *dev)
16647 {
16648         struct drm_i915_private *dev_priv = to_i915(dev);
16649         enum pipe pipe;
16650         struct intel_crtc *crtc;
16651         struct intel_encoder *encoder;
16652         struct intel_connector *connector;
16653         struct drm_connector_list_iter conn_iter;
16654         int i;
16655
16656         dev_priv->active_crtcs = 0;
16657
16658         for_each_intel_crtc(dev, crtc) {
16659                 struct intel_crtc_state *crtc_state =
16660                         to_intel_crtc_state(crtc->base.state);
16661
16662                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
16663                 memset(crtc_state, 0, sizeof(*crtc_state));
16664                 __drm_atomic_helper_crtc_reset(&crtc->base, &crtc_state->base);
16665
16666                 crtc_state->base.active = crtc_state->base.enable =
16667                         dev_priv->display.get_pipe_config(crtc, crtc_state);
16668
16669                 crtc->base.enabled = crtc_state->base.enable;
16670                 crtc->active = crtc_state->base.active;
16671
16672                 if (crtc_state->base.active)
16673                         dev_priv->active_crtcs |= 1 << crtc->pipe;
16674
16675                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16676                               crtc->base.base.id, crtc->base.name,
16677                               enableddisabled(crtc_state->base.active));
16678         }
16679
16680         readout_plane_state(dev_priv);
16681
16682         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16683                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16684
16685                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
16686                                                         &pll->state.hw_state);
16687
16688                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
16689                     pll->info->id == DPLL_ID_EHL_DPLL4) {
16690                         pll->wakeref = intel_display_power_get(dev_priv,
16691                                                                POWER_DOMAIN_DPLL_DC_OFF);
16692                 }
16693
16694                 pll->state.crtc_mask = 0;
16695                 for_each_intel_crtc(dev, crtc) {
16696                         struct intel_crtc_state *crtc_state =
16697                                 to_intel_crtc_state(crtc->base.state);
16698
16699                         if (crtc_state->base.active &&
16700                             crtc_state->shared_dpll == pll)
16701                                 pll->state.crtc_mask |= 1 << crtc->pipe;
16702                 }
16703                 pll->active_mask = pll->state.crtc_mask;
16704
16705                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
16706                               pll->info->name, pll->state.crtc_mask, pll->on);
16707         }
16708
16709         for_each_intel_encoder(dev, encoder) {
16710                 pipe = 0;
16711
16712                 if (encoder->get_hw_state(encoder, &pipe)) {
16713                         struct intel_crtc_state *crtc_state;
16714
16715                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16716                         crtc_state = to_intel_crtc_state(crtc->base.state);
16717
16718                         encoder->base.crtc = &crtc->base;
16719                         encoder->get_config(encoder, crtc_state);
16720                 } else {
16721                         encoder->base.crtc = NULL;
16722                 }
16723
16724                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16725                               encoder->base.base.id, encoder->base.name,
16726                               enableddisabled(encoder->base.crtc),
16727                               pipe_name(pipe));
16728         }
16729
16730         drm_connector_list_iter_begin(dev, &conn_iter);
16731         for_each_intel_connector_iter(connector, &conn_iter) {
16732                 if (connector->get_hw_state(connector)) {
16733                         connector->base.dpms = DRM_MODE_DPMS_ON;
16734
16735                         encoder = connector->encoder;
16736                         connector->base.encoder = &encoder->base;
16737
16738                         if (encoder->base.crtc &&
16739                             encoder->base.crtc->state->active) {
16740                                 /*
16741                                  * This has to be done during hardware readout
16742                                  * because anything calling .crtc_disable may
16743                                  * rely on the connector_mask being accurate.
16744                                  */
16745                                 encoder->base.crtc->state->connector_mask |=
16746                                         drm_connector_mask(&connector->base);
16747                                 encoder->base.crtc->state->encoder_mask |=
16748                                         drm_encoder_mask(&encoder->base);
16749                         }
16750
16751                 } else {
16752                         connector->base.dpms = DRM_MODE_DPMS_OFF;
16753                         connector->base.encoder = NULL;
16754                 }
16755                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16756                               connector->base.base.id, connector->base.name,
16757                               enableddisabled(connector->base.encoder));
16758         }
16759         drm_connector_list_iter_end(&conn_iter);
16760
16761         for_each_intel_crtc(dev, crtc) {
16762                 struct intel_bw_state *bw_state =
16763                         to_intel_bw_state(dev_priv->bw_obj.state);
16764                 struct intel_crtc_state *crtc_state =
16765                         to_intel_crtc_state(crtc->base.state);
16766                 struct intel_plane *plane;
16767                 int min_cdclk = 0;
16768
16769                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
16770                 if (crtc_state->base.active) {
16771                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
16772                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
16773                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
16774                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
16775                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
16776
16777                         /*
16778                          * The initial mode needs to be set in order to keep
16779                          * the atomic core happy. It wants a valid mode if the
16780                          * crtc's enabled, so we do the above call.
16781                          *
16782                          * But we don't set all the derived state fully, hence
16783                          * set a flag to indicate that a full recalculation is
16784                          * needed on the next commit.
16785                          */
16786                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
16787
16788                         intel_crtc_compute_pixel_rate(crtc_state);
16789
16790                         if (dev_priv->display.modeset_calc_cdclk) {
16791                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
16792                                 if (WARN_ON(min_cdclk < 0))
16793                                         min_cdclk = 0;
16794                         }
16795
16796                         drm_calc_timestamping_constants(&crtc->base,
16797                                                         &crtc_state->base.adjusted_mode);
16798                         update_scanline_offset(crtc_state);
16799                 }
16800
16801                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
16802                 dev_priv->min_voltage_level[crtc->pipe] =
16803                         crtc_state->min_voltage_level;
16804
16805                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
16806                         const struct intel_plane_state *plane_state =
16807                                 to_intel_plane_state(plane->base.state);
16808
16809                         /*
16810                          * FIXME don't have the fb yet, so can't
16811                          * use intel_plane_data_rate() :(
16812                          */
16813                         if (plane_state->base.visible)
16814                                 crtc_state->data_rate[plane->id] =
16815                                         4 * crtc_state->pixel_rate;
16816                 }
16817
16818                 intel_bw_crtc_update(bw_state, crtc_state);
16819
16820                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
16821         }
16822 }
16823
16824 static void
16825 get_encoder_power_domains(struct drm_i915_private *dev_priv)
16826 {
16827         struct intel_encoder *encoder;
16828
16829         for_each_intel_encoder(&dev_priv->drm, encoder) {
16830                 struct intel_crtc_state *crtc_state;
16831
16832                 if (!encoder->get_power_domains)
16833                         continue;
16834
16835                 /*
16836                  * MST-primary and inactive encoders don't have a crtc state
16837                  * and neither of these require any power domain references.
16838                  */
16839                 if (!encoder->base.crtc)
16840                         continue;
16841
16842                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
16843                 encoder->get_power_domains(encoder, crtc_state);
16844         }
16845 }
16846
16847 static void intel_early_display_was(struct drm_i915_private *dev_priv)
16848 {
16849         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
16850         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
16851                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
16852                            DARBF_GATING_DIS);
16853
16854         if (IS_HASWELL(dev_priv)) {
16855                 /*
16856                  * WaRsPkgCStateDisplayPMReq:hsw
16857                  * System hang if this isn't done before disabling all planes!
16858                  */
16859                 I915_WRITE(CHICKEN_PAR1_1,
16860                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
16861         }
16862 }
16863
16864 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
16865                                        enum port port, i915_reg_t hdmi_reg)
16866 {
16867         u32 val = I915_READ(hdmi_reg);
16868
16869         if (val & SDVO_ENABLE ||
16870             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
16871                 return;
16872
16873         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
16874                       port_name(port));
16875
16876         val &= ~SDVO_PIPE_SEL_MASK;
16877         val |= SDVO_PIPE_SEL(PIPE_A);
16878
16879         I915_WRITE(hdmi_reg, val);
16880 }
16881
16882 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
16883                                      enum port port, i915_reg_t dp_reg)
16884 {
16885         u32 val = I915_READ(dp_reg);
16886
16887         if (val & DP_PORT_EN ||
16888             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
16889                 return;
16890
16891         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
16892                       port_name(port));
16893
16894         val &= ~DP_PIPE_SEL_MASK;
16895         val |= DP_PIPE_SEL(PIPE_A);
16896
16897         I915_WRITE(dp_reg, val);
16898 }
16899
16900 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
16901 {
16902         /*
16903          * The BIOS may select transcoder B on some of the PCH
16904          * ports even it doesn't enable the port. This would trip
16905          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
16906          * Sanitize the transcoder select bits to prevent that. We
16907          * assume that the BIOS never actually enabled the port,
16908          * because if it did we'd actually have to toggle the port
16909          * on and back off to make the transcoder A select stick
16910          * (see. intel_dp_link_down(), intel_disable_hdmi(),
16911          * intel_disable_sdvo()).
16912          */
16913         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
16914         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
16915         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
16916
16917         /* PCH SDVOB multiplex with HDMIB */
16918         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
16919         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
16920         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
16921 }
16922
16923 /* Scan out the current hw modeset state,
16924  * and sanitizes it to the current state
16925  */
16926 static void
16927 intel_modeset_setup_hw_state(struct drm_device *dev,
16928                              struct drm_modeset_acquire_ctx *ctx)
16929 {
16930         struct drm_i915_private *dev_priv = to_i915(dev);
16931         struct intel_crtc_state *crtc_state;
16932         struct intel_encoder *encoder;
16933         struct intel_crtc *crtc;
16934         intel_wakeref_t wakeref;
16935         int i;
16936
16937         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
16938
16939         intel_early_display_was(dev_priv);
16940         intel_modeset_readout_hw_state(dev);
16941
16942         /* HW state is read out, now we need to sanitize this mess. */
16943
16944         /* Sanitize the TypeC port mode upfront, encoders depend on this */
16945         for_each_intel_encoder(dev, encoder) {
16946                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
16947
16948                 /* We need to sanitize only the MST primary port. */
16949                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
16950                     intel_phy_is_tc(dev_priv, phy))
16951                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
16952         }
16953
16954         get_encoder_power_domains(dev_priv);
16955
16956         if (HAS_PCH_IBX(dev_priv))
16957                 ibx_sanitize_pch_ports(dev_priv);
16958
16959         /*
16960          * intel_sanitize_plane_mapping() may need to do vblank
16961          * waits, so we need vblank interrupts restored beforehand.
16962          */
16963         for_each_intel_crtc(&dev_priv->drm, crtc) {
16964                 crtc_state = to_intel_crtc_state(crtc->base.state);
16965
16966                 drm_crtc_vblank_reset(&crtc->base);
16967
16968                 if (crtc_state->base.active)
16969                         intel_crtc_vblank_on(crtc_state);
16970         }
16971
16972         intel_sanitize_plane_mapping(dev_priv);
16973
16974         for_each_intel_encoder(dev, encoder)
16975                 intel_sanitize_encoder(encoder);
16976
16977         for_each_intel_crtc(&dev_priv->drm, crtc) {
16978                 crtc_state = to_intel_crtc_state(crtc->base.state);
16979                 intel_sanitize_crtc(crtc, ctx);
16980                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
16981         }
16982
16983         intel_modeset_update_connector_atomic_state(dev);
16984
16985         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16986                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16987
16988                 if (!pll->on || pll->active_mask)
16989                         continue;
16990
16991                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
16992                               pll->info->name);
16993
16994                 pll->info->funcs->disable(dev_priv, pll);
16995                 pll->on = false;
16996         }
16997
16998         if (IS_G4X(dev_priv)) {
16999                 g4x_wm_get_hw_state(dev_priv);
17000                 g4x_wm_sanitize(dev_priv);
17001         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
17002                 vlv_wm_get_hw_state(dev_priv);
17003                 vlv_wm_sanitize(dev_priv);
17004         } else if (INTEL_GEN(dev_priv) >= 9) {
17005                 skl_wm_get_hw_state(dev_priv);
17006         } else if (HAS_PCH_SPLIT(dev_priv)) {
17007                 ilk_wm_get_hw_state(dev_priv);
17008         }
17009
17010         for_each_intel_crtc(dev, crtc) {
17011                 u64 put_domains;
17012
17013                 crtc_state = to_intel_crtc_state(crtc->base.state);
17014                 put_domains = modeset_get_crtc_power_domains(crtc_state);
17015                 if (WARN_ON(put_domains))
17016                         modeset_put_power_domains(dev_priv, put_domains);
17017         }
17018
17019         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
17020
17021         intel_fbc_init_pipe_state(dev_priv);
17022 }
17023
17024 void intel_display_resume(struct drm_device *dev)
17025 {
17026         struct drm_i915_private *dev_priv = to_i915(dev);
17027         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
17028         struct drm_modeset_acquire_ctx ctx;
17029         int ret;
17030
17031         dev_priv->modeset_restore_state = NULL;
17032         if (state)
17033                 state->acquire_ctx = &ctx;
17034
17035         drm_modeset_acquire_init(&ctx, 0);
17036
17037         while (1) {
17038                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
17039                 if (ret != -EDEADLK)
17040                         break;
17041
17042                 drm_modeset_backoff(&ctx);
17043         }
17044
17045         if (!ret)
17046                 ret = __intel_display_resume(dev, state, &ctx);
17047
17048         intel_enable_ipc(dev_priv);
17049         drm_modeset_drop_locks(&ctx);
17050         drm_modeset_acquire_fini(&ctx);
17051
17052         if (ret)
17053                 DRM_ERROR("Restoring old state failed with %i\n", ret);
17054         if (state)
17055                 drm_atomic_state_put(state);
17056 }
17057
17058 static void intel_hpd_poll_fini(struct drm_device *dev)
17059 {
17060         struct intel_connector *connector;
17061         struct drm_connector_list_iter conn_iter;
17062
17063         /* Kill all the work that may have been queued by hpd. */
17064         drm_connector_list_iter_begin(dev, &conn_iter);
17065         for_each_intel_connector_iter(connector, &conn_iter) {
17066                 if (connector->modeset_retry_work.func)
17067                         cancel_work_sync(&connector->modeset_retry_work);
17068                 if (connector->hdcp.shim) {
17069                         cancel_delayed_work_sync(&connector->hdcp.check_work);
17070                         cancel_work_sync(&connector->hdcp.prop_work);
17071                 }
17072         }
17073         drm_connector_list_iter_end(&conn_iter);
17074 }
17075
17076 void intel_modeset_cleanup(struct drm_device *dev)
17077 {
17078         struct drm_i915_private *dev_priv = to_i915(dev);
17079
17080         flush_workqueue(dev_priv->modeset_wq);
17081
17082         flush_work(&dev_priv->atomic_helper.free_work);
17083         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
17084
17085         /*
17086          * Interrupts and polling as the first thing to avoid creating havoc.
17087          * Too much stuff here (turning of connectors, ...) would
17088          * experience fancy races otherwise.
17089          */
17090         intel_irq_uninstall(dev_priv);
17091
17092         /*
17093          * Due to the hpd irq storm handling the hotplug work can re-arm the
17094          * poll handlers. Hence disable polling after hpd handling is shut down.
17095          */
17096         intel_hpd_poll_fini(dev);
17097
17098         /* poll work can call into fbdev, hence clean that up afterwards */
17099         intel_fbdev_fini(dev_priv);
17100
17101         intel_unregister_dsm_handler();
17102
17103         intel_fbc_global_disable(dev_priv);
17104
17105         /* flush any delayed tasks or pending work */
17106         flush_scheduled_work();
17107
17108         intel_hdcp_component_fini(dev_priv);
17109
17110         drm_mode_config_cleanup(dev);
17111
17112         intel_overlay_cleanup(dev_priv);
17113
17114         intel_gmbus_teardown(dev_priv);
17115
17116         destroy_workqueue(dev_priv->modeset_wq);
17117
17118         intel_fbc_cleanup_cfb(dev_priv);
17119 }
17120
17121 /*
17122  * set vga decode state - true == enable VGA decode
17123  */
17124 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
17125 {
17126         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17127         u16 gmch_ctrl;
17128
17129         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
17130                 DRM_ERROR("failed to read control word\n");
17131                 return -EIO;
17132         }
17133
17134         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
17135                 return 0;
17136
17137         if (state)
17138                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
17139         else
17140                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
17141
17142         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
17143                 DRM_ERROR("failed to write control word\n");
17144                 return -EIO;
17145         }
17146
17147         return 0;
17148 }
17149
17150 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
17151
17152 struct intel_display_error_state {
17153
17154         u32 power_well_driver;
17155
17156         struct intel_cursor_error_state {
17157                 u32 control;
17158                 u32 position;
17159                 u32 base;
17160                 u32 size;
17161         } cursor[I915_MAX_PIPES];
17162
17163         struct intel_pipe_error_state {
17164                 bool power_domain_on;
17165                 u32 source;
17166                 u32 stat;
17167         } pipe[I915_MAX_PIPES];
17168
17169         struct intel_plane_error_state {
17170                 u32 control;
17171                 u32 stride;
17172                 u32 size;
17173                 u32 pos;
17174                 u32 addr;
17175                 u32 surface;
17176                 u32 tile_offset;
17177         } plane[I915_MAX_PIPES];
17178
17179         struct intel_transcoder_error_state {
17180                 bool available;
17181                 bool power_domain_on;
17182                 enum transcoder cpu_transcoder;
17183
17184                 u32 conf;
17185
17186                 u32 htotal;
17187                 u32 hblank;
17188                 u32 hsync;
17189                 u32 vtotal;
17190                 u32 vblank;
17191                 u32 vsync;
17192         } transcoder[4];
17193 };
17194
17195 struct intel_display_error_state *
17196 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
17197 {
17198         struct intel_display_error_state *error;
17199         int transcoders[] = {
17200                 TRANSCODER_A,
17201                 TRANSCODER_B,
17202                 TRANSCODER_C,
17203                 TRANSCODER_EDP,
17204         };
17205         int i;
17206
17207         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
17208
17209         if (!HAS_DISPLAY(dev_priv))
17210                 return NULL;
17211
17212         error = kzalloc(sizeof(*error), GFP_ATOMIC);
17213         if (error == NULL)
17214                 return NULL;
17215
17216         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17217                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
17218
17219         for_each_pipe(dev_priv, i) {
17220                 error->pipe[i].power_domain_on =
17221                         __intel_display_power_is_enabled(dev_priv,
17222                                                          POWER_DOMAIN_PIPE(i));
17223                 if (!error->pipe[i].power_domain_on)
17224                         continue;
17225
17226                 error->cursor[i].control = I915_READ(CURCNTR(i));
17227                 error->cursor[i].position = I915_READ(CURPOS(i));
17228                 error->cursor[i].base = I915_READ(CURBASE(i));
17229
17230                 error->plane[i].control = I915_READ(DSPCNTR(i));
17231                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
17232                 if (INTEL_GEN(dev_priv) <= 3) {
17233                         error->plane[i].size = I915_READ(DSPSIZE(i));
17234                         error->plane[i].pos = I915_READ(DSPPOS(i));
17235                 }
17236                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17237                         error->plane[i].addr = I915_READ(DSPADDR(i));
17238                 if (INTEL_GEN(dev_priv) >= 4) {
17239                         error->plane[i].surface = I915_READ(DSPSURF(i));
17240                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
17241                 }
17242
17243                 error->pipe[i].source = I915_READ(PIPESRC(i));
17244
17245                 if (HAS_GMCH(dev_priv))
17246                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
17247         }
17248
17249         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17250                 enum transcoder cpu_transcoder = transcoders[i];
17251
17252                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
17253                         continue;
17254
17255                 error->transcoder[i].available = true;
17256                 error->transcoder[i].power_domain_on =
17257                         __intel_display_power_is_enabled(dev_priv,
17258                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
17259                 if (!error->transcoder[i].power_domain_on)
17260                         continue;
17261
17262                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
17263
17264                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
17265                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
17266                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
17267                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
17268                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
17269                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
17270                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
17271         }
17272
17273         return error;
17274 }
17275
17276 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
17277
17278 void
17279 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17280                                 struct intel_display_error_state *error)
17281 {
17282         struct drm_i915_private *dev_priv = m->i915;
17283         int i;
17284
17285         if (!error)
17286                 return;
17287
17288         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
17289         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17290                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
17291                            error->power_well_driver);
17292         for_each_pipe(dev_priv, i) {
17293                 err_printf(m, "Pipe [%d]:\n", i);
17294                 err_printf(m, "  Power: %s\n",
17295                            onoff(error->pipe[i].power_domain_on));
17296                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
17297                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
17298
17299                 err_printf(m, "Plane [%d]:\n", i);
17300                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
17301                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
17302                 if (INTEL_GEN(dev_priv) <= 3) {
17303                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
17304                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
17305                 }
17306                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17307                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
17308                 if (INTEL_GEN(dev_priv) >= 4) {
17309                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
17310                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
17311                 }
17312
17313                 err_printf(m, "Cursor [%d]:\n", i);
17314                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
17315                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
17316                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
17317         }
17318
17319         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17320                 if (!error->transcoder[i].available)
17321                         continue;
17322
17323                 err_printf(m, "CPU transcoder: %s\n",
17324                            transcoder_name(error->transcoder[i].cpu_transcoder));
17325                 err_printf(m, "  Power: %s\n",
17326                            onoff(error->transcoder[i].power_domain_on));
17327                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
17328                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
17329                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
17330                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
17331                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
17332                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
17333                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
17334         }
17335 }
17336
17337 #endif