drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34 #include <linux/vgaarb.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45 #include <drm/i915_drm.h>
  46
  47 #include "display/intel_crt.h"
  48 #include "display/intel_ddi.h"
  49 #include "display/intel_dp.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "i915_drv.h"
  60 #include "i915_trace.h"
  61 #include "intel_acpi.h"
  62 #include "intel_atomic.h"
  63 #include "intel_atomic_plane.h"
  64 #include "intel_bw.h"
  65 #include "intel_cdclk.h"
  66 #include "intel_color.h"
  67 #include "intel_display_types.h"
  68 #include "intel_fbc.h"
  69 #include "intel_fbdev.h"
  70 #include "intel_fifo_underrun.h"
  71 #include "intel_frontbuffer.h"
  72 #include "intel_hdcp.h"
  73 #include "intel_hotplug.h"
  74 #include "intel_overlay.h"
  75 #include "intel_pipe_crc.h"
  76 #include "intel_pm.h"
  77 #include "intel_psr.h"
  78 #include "intel_quirks.h"
  79 #include "intel_sideband.h"
  80 #include "intel_sprite.h"
  81 #include "intel_tc.h"
  82
  83 /* Primary plane formats for gen <= 3 */
  84 static const u32 i8xx_primary_formats[] = {
  85         DRM_FORMAT_C8,
  86         DRM_FORMAT_RGB565,
  87         DRM_FORMAT_XRGB1555,
  88         DRM_FORMAT_XRGB8888,
  89 };
  90
  91 /* Primary plane formats for gen >= 4 */
  92 static const u32 i965_primary_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_XRGB2101010,
  98         DRM_FORMAT_XBGR2101010,
  99 };
 100
 101 static const u64 i9xx_format_modifiers[] = {
 102         I915_FORMAT_MOD_X_TILED,
 103         DRM_FORMAT_MOD_LINEAR,
 104         DRM_FORMAT_MOD_INVALID
 105 };
 106
 107 /* Cursor formats */
 108 static const u32 intel_cursor_formats[] = {
 109         DRM_FORMAT_ARGB8888,
 110 };
 111
 112 static const u64 cursor_format_modifiers[] = {
 113         DRM_FORMAT_MOD_LINEAR,
 114         DRM_FORMAT_MOD_INVALID
 115 };
 116
 117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 118                                 struct intel_crtc_state *pipe_config);
 119 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 120                                    struct intel_crtc_state *pipe_config);
 121
 122 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 123                                   struct drm_i915_gem_object *obj,
 124                                   struct drm_mode_fb_cmd2 *mode_cmd);
 125 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 126 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 127 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 128                                          const struct intel_link_m_n *m_n,
 129                                          const struct intel_link_m_n *m2_n2);
 130 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 131 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 132 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 133 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 134 static void vlv_prepare_pll(struct intel_crtc *crtc,
 135                             const struct intel_crtc_state *pipe_config);
 136 static void chv_prepare_pll(struct intel_crtc *crtc,
 137                             const struct intel_crtc_state *pipe_config);
 138 static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 139 static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 140 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 141                                     struct intel_crtc_state *crtc_state);
 142 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 143 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 144 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 145 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 146                                          struct drm_modeset_acquire_ctx *ctx);
 147 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 148
 149 struct intel_limit {
 150         struct {
 151                 int min, max;
 152         } dot, vco, n, m, m1, m2, p, p1;
 153
 154         struct {
 155                 int dot_limit;
 156                 int p2_slow, p2_fast;
 157         } p2;
 158 };
 159
 160 /* returns HPLL frequency in kHz */
 161 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 162 {
 163         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 164
 165         /* Obtain SKU information */
 166         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 167                 CCK_FUSE_HPLL_FREQ_MASK;
 168
 169         return vco_freq[hpll_freq] * 1000;
 170 }
 171
 172 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 173                       const char *name, u32 reg, int ref_freq)
 174 {
 175         u32 val;
 176         int divider;
 177
 178         val = vlv_cck_read(dev_priv, reg);
 179         divider = val & CCK_FREQUENCY_VALUES;
 180
 181         WARN((val & CCK_FREQUENCY_STATUS) !=
 182              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 183              "%s change in progress\n", name);
 184
 185         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 186 }
 187
 188 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 189                            const char *name, u32 reg)
 190 {
 191         int hpll;
 192
 193         vlv_cck_get(dev_priv);
 194
 195         if (dev_priv->hpll_freq == 0)
 196                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 197
 198         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 199
 200         vlv_cck_put(dev_priv);
 201
 202         return hpll;
 203 }
 204
 205 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 206 {
 207         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 208                 return;
 209
 210         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 211                                                       CCK_CZ_CLOCK_CONTROL);
 212
 213         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 214 }
 215
 216 static inline u32 /* units of 100MHz */
 217 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 218                     const struct intel_crtc_state *pipe_config)
 219 {
 220         if (HAS_DDI(dev_priv))
 221                 return pipe_config->port_clock; /* SPLL */
 222         else
 223                 return dev_priv->fdi_pll_freq;
 224 }
 225
 226 static const struct intel_limit intel_limits_i8xx_dac = {
 227         .dot = { .min = 25000, .max = 350000 },
 228         .vco = { .min = 908000, .max = 1512000 },
 229         .n = { .min = 2, .max = 16 },
 230         .m = { .min = 96, .max = 140 },
 231         .m1 = { .min = 18, .max = 26 },
 232         .m2 = { .min = 6, .max = 16 },
 233         .p = { .min = 4, .max = 128 },
 234         .p1 = { .min = 2, .max = 33 },
 235         .p2 = { .dot_limit = 165000,
 236                 .p2_slow = 4, .p2_fast = 2 },
 237 };
 238
 239 static const struct intel_limit intel_limits_i8xx_dvo = {
 240         .dot = { .min = 25000, .max = 350000 },
 241         .vco = { .min = 908000, .max = 1512000 },
 242         .n = { .min = 2, .max = 16 },
 243         .m = { .min = 96, .max = 140 },
 244         .m1 = { .min = 18, .max = 26 },
 245         .m2 = { .min = 6, .max = 16 },
 246         .p = { .min = 4, .max = 128 },
 247         .p1 = { .min = 2, .max = 33 },
 248         .p2 = { .dot_limit = 165000,
 249                 .p2_slow = 4, .p2_fast = 4 },
 250 };
 251
 252 static const struct intel_limit intel_limits_i8xx_lvds = {
 253         .dot = { .min = 25000, .max = 350000 },
 254         .vco = { .min = 908000, .max = 1512000 },
 255         .n = { .min = 2, .max = 16 },
 256         .m = { .min = 96, .max = 140 },
 257         .m1 = { .min = 18, .max = 26 },
 258         .m2 = { .min = 6, .max = 16 },
 259         .p = { .min = 4, .max = 128 },
 260         .p1 = { .min = 1, .max = 6 },
 261         .p2 = { .dot_limit = 165000,
 262                 .p2_slow = 14, .p2_fast = 7 },
 263 };
 264
 265 static const struct intel_limit intel_limits_i9xx_sdvo = {
 266         .dot = { .min = 20000, .max = 400000 },
 267         .vco = { .min = 1400000, .max = 2800000 },
 268         .n = { .min = 1, .max = 6 },
 269         .m = { .min = 70, .max = 120 },
 270         .m1 = { .min = 8, .max = 18 },
 271         .m2 = { .min = 3, .max = 7 },
 272         .p = { .min = 5, .max = 80 },
 273         .p1 = { .min = 1, .max = 8 },
 274         .p2 = { .dot_limit = 200000,
 275                 .p2_slow = 10, .p2_fast = 5 },
 276 };
 277
 278 static const struct intel_limit intel_limits_i9xx_lvds = {
 279         .dot = { .min = 20000, .max = 400000 },
 280         .vco = { .min = 1400000, .max = 2800000 },
 281         .n = { .min = 1, .max = 6 },
 282         .m = { .min = 70, .max = 120 },
 283         .m1 = { .min = 8, .max = 18 },
 284         .m2 = { .min = 3, .max = 7 },
 285         .p = { .min = 7, .max = 98 },
 286         .p1 = { .min = 1, .max = 8 },
 287         .p2 = { .dot_limit = 112000,
 288                 .p2_slow = 14, .p2_fast = 7 },
 289 };
 290
 291
 292 static const struct intel_limit intel_limits_g4x_sdvo = {
 293         .dot = { .min = 25000, .max = 270000 },
 294         .vco = { .min = 1750000, .max = 3500000},
 295         .n = { .min = 1, .max = 4 },
 296         .m = { .min = 104, .max = 138 },
 297         .m1 = { .min = 17, .max = 23 },
 298         .m2 = { .min = 5, .max = 11 },
 299         .p = { .min = 10, .max = 30 },
 300         .p1 = { .min = 1, .max = 3},
 301         .p2 = { .dot_limit = 270000,
 302                 .p2_slow = 10,
 303                 .p2_fast = 10
 304         },
 305 };
 306
 307 static const struct intel_limit intel_limits_g4x_hdmi = {
 308         .dot = { .min = 22000, .max = 400000 },
 309         .vco = { .min = 1750000, .max = 3500000},
 310         .n = { .min = 1, .max = 4 },
 311         .m = { .min = 104, .max = 138 },
 312         .m1 = { .min = 16, .max = 23 },
 313         .m2 = { .min = 5, .max = 11 },
 314         .p = { .min = 5, .max = 80 },
 315         .p1 = { .min = 1, .max = 8},
 316         .p2 = { .dot_limit = 165000,
 317                 .p2_slow = 10, .p2_fast = 5 },
 318 };
 319
 320 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 321         .dot = { .min = 20000, .max = 115000 },
 322         .vco = { .min = 1750000, .max = 3500000 },
 323         .n = { .min = 1, .max = 3 },
 324         .m = { .min = 104, .max = 138 },
 325         .m1 = { .min = 17, .max = 23 },
 326         .m2 = { .min = 5, .max = 11 },
 327         .p = { .min = 28, .max = 112 },
 328         .p1 = { .min = 2, .max = 8 },
 329         .p2 = { .dot_limit = 0,
 330                 .p2_slow = 14, .p2_fast = 14
 331         },
 332 };
 333
 334 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 335         .dot = { .min = 80000, .max = 224000 },
 336         .vco = { .min = 1750000, .max = 3500000 },
 337         .n = { .min = 1, .max = 3 },
 338         .m = { .min = 104, .max = 138 },
 339         .m1 = { .min = 17, .max = 23 },
 340         .m2 = { .min = 5, .max = 11 },
 341         .p = { .min = 14, .max = 42 },
 342         .p1 = { .min = 2, .max = 6 },
 343         .p2 = { .dot_limit = 0,
 344                 .p2_slow = 7, .p2_fast = 7
 345         },
 346 };
 347
 348 static const struct intel_limit intel_limits_pineview_sdvo = {
 349         .dot = { .min = 20000, .max = 400000},
 350         .vco = { .min = 1700000, .max = 3500000 },
 351         /* Pineview's Ncounter is a ring counter */
 352         .n = { .min = 3, .max = 6 },
 353         .m = { .min = 2, .max = 256 },
 354         /* Pineview only has one combined m divider, which we treat as m2. */
 355         .m1 = { .min = 0, .max = 0 },
 356         .m2 = { .min = 0, .max = 254 },
 357         .p = { .min = 5, .max = 80 },
 358         .p1 = { .min = 1, .max = 8 },
 359         .p2 = { .dot_limit = 200000,
 360                 .p2_slow = 10, .p2_fast = 5 },
 361 };
 362
 363 static const struct intel_limit intel_limits_pineview_lvds = {
 364         .dot = { .min = 20000, .max = 400000 },
 365         .vco = { .min = 1700000, .max = 3500000 },
 366         .n = { .min = 3, .max = 6 },
 367         .m = { .min = 2, .max = 256 },
 368         .m1 = { .min = 0, .max = 0 },
 369         .m2 = { .min = 0, .max = 254 },
 370         .p = { .min = 7, .max = 112 },
 371         .p1 = { .min = 1, .max = 8 },
 372         .p2 = { .dot_limit = 112000,
 373                 .p2_slow = 14, .p2_fast = 14 },
 374 };
 375
 376 /* Ironlake / Sandybridge
 377  *
 378  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 379  * the range value for them is (actual_value - 2).
 380  */
 381 static const struct intel_limit intel_limits_ironlake_dac = {
 382         .dot = { .min = 25000, .max = 350000 },
 383         .vco = { .min = 1760000, .max = 3510000 },
 384         .n = { .min = 1, .max = 5 },
 385         .m = { .min = 79, .max = 127 },
 386         .m1 = { .min = 12, .max = 22 },
 387         .m2 = { .min = 5, .max = 9 },
 388         .p = { .min = 5, .max = 80 },
 389         .p1 = { .min = 1, .max = 8 },
 390         .p2 = { .dot_limit = 225000,
 391                 .p2_slow = 10, .p2_fast = 5 },
 392 };
 393
 394 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 395         .dot = { .min = 25000, .max = 350000 },
 396         .vco = { .min = 1760000, .max = 3510000 },
 397         .n = { .min = 1, .max = 3 },
 398         .m = { .min = 79, .max = 118 },
 399         .m1 = { .min = 12, .max = 22 },
 400         .m2 = { .min = 5, .max = 9 },
 401         .p = { .min = 28, .max = 112 },
 402         .p1 = { .min = 2, .max = 8 },
 403         .p2 = { .dot_limit = 225000,
 404                 .p2_slow = 14, .p2_fast = 14 },
 405 };
 406
 407 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 408         .dot = { .min = 25000, .max = 350000 },
 409         .vco = { .min = 1760000, .max = 3510000 },
 410         .n = { .min = 1, .max = 3 },
 411         .m = { .min = 79, .max = 127 },
 412         .m1 = { .min = 12, .max = 22 },
 413         .m2 = { .min = 5, .max = 9 },
 414         .p = { .min = 14, .max = 56 },
 415         .p1 = { .min = 2, .max = 8 },
 416         .p2 = { .dot_limit = 225000,
 417                 .p2_slow = 7, .p2_fast = 7 },
 418 };
 419
 420 /* LVDS 100mhz refclk limits. */
 421 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 422         .dot = { .min = 25000, .max = 350000 },
 423         .vco = { .min = 1760000, .max = 3510000 },
 424         .n = { .min = 1, .max = 2 },
 425         .m = { .min = 79, .max = 126 },
 426         .m1 = { .min = 12, .max = 22 },
 427         .m2 = { .min = 5, .max = 9 },
 428         .p = { .min = 28, .max = 112 },
 429         .p1 = { .min = 2, .max = 8 },
 430         .p2 = { .dot_limit = 225000,
 431                 .p2_slow = 14, .p2_fast = 14 },
 432 };
 433
 434 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 435         .dot = { .min = 25000, .max = 350000 },
 436         .vco = { .min = 1760000, .max = 3510000 },
 437         .n = { .min = 1, .max = 3 },
 438         .m = { .min = 79, .max = 126 },
 439         .m1 = { .min = 12, .max = 22 },
 440         .m2 = { .min = 5, .max = 9 },
 441         .p = { .min = 14, .max = 42 },
 442         .p1 = { .min = 2, .max = 6 },
 443         .p2 = { .dot_limit = 225000,
 444                 .p2_slow = 7, .p2_fast = 7 },
 445 };
 446
 447 static const struct intel_limit intel_limits_vlv = {
 448          /*
 449           * These are the data rate limits (measured in fast clocks)
 450           * since those are the strictest limits we have. The fast
 451           * clock and actual rate limits are more relaxed, so checking
 452           * them would make no difference.
 453           */
 454         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 455         .vco = { .min = 4000000, .max = 6000000 },
 456         .n = { .min = 1, .max = 7 },
 457         .m1 = { .min = 2, .max = 3 },
 458         .m2 = { .min = 11, .max = 156 },
 459         .p1 = { .min = 2, .max = 3 },
 460         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 461 };
 462
 463 static const struct intel_limit intel_limits_chv = {
 464         /*
 465          * These are the data rate limits (measured in fast clocks)
 466          * since those are the strictest limits we have.  The fast
 467          * clock and actual rate limits are more relaxed, so checking
 468          * them would make no difference.
 469          */
 470         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 471         .vco = { .min = 4800000, .max = 6480000 },
 472         .n = { .min = 1, .max = 1 },
 473         .m1 = { .min = 2, .max = 2 },
 474         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 475         .p1 = { .min = 2, .max = 4 },
 476         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 477 };
 478
 479 static const struct intel_limit intel_limits_bxt = {
 480         /* FIXME: find real dot limits */
 481         .dot = { .min = 0, .max = INT_MAX },
 482         .vco = { .min = 4800000, .max = 6700000 },
 483         .n = { .min = 1, .max = 1 },
 484         .m1 = { .min = 2, .max = 2 },
 485         /* FIXME: find real m2 limits */
 486         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 487         .p1 = { .min = 2, .max = 4 },
 488         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 489 };
 490
 491 /* WA Display #0827: Gen9:all */
 492 static void
 493 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 494 {
 495         if (enable)
 496                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 497                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 498                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 499         else
 500                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 501                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 502                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 503 }
 504
 505 /* Wa_2006604312:icl */
 506 static void
 507 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 508                        bool enable)
 509 {
 510         if (enable)
 511                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 512                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 513         else
 514                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 515                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 516 }
 517
 518 static bool
 519 needs_modeset(const struct intel_crtc_state *state)
 520 {
 521         return drm_atomic_crtc_needs_modeset(&state->base);
 522 }
 523
 524 /*
 525  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 526  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 527  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 528  * The helpers' return value is the rate of the clock that is fed to the
 529  * display engine's pipe which can be the above fast dot clock rate or a
 530  * divided-down version of it.
 531  */
 532 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 533 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 534 {
 535         clock->m = clock->m2 + 2;
 536         clock->p = clock->p1 * clock->p2;
 537         if (WARN_ON(clock->n == 0 || clock->p == 0))
 538                 return 0;
 539         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 540         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 541
 542         return clock->dot;
 543 }
 544
 545 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 546 {
 547         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 548 }
 549
 550 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = i9xx_dpll_compute_m(clock);
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 563 {
 564         clock->m = clock->m1 * clock->m2;
 565         clock->p = clock->p1 * clock->p2;
 566         if (WARN_ON(clock->n == 0 || clock->p == 0))
 567                 return 0;
 568         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 569         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 570
 571         return clock->dot / 5;
 572 }
 573
 574 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 575 {
 576         clock->m = clock->m1 * clock->m2;
 577         clock->p = clock->p1 * clock->p2;
 578         if (WARN_ON(clock->n == 0 || clock->p == 0))
 579                 return 0;
 580         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 581                                            clock->n << 22);
 582         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 583
 584         return clock->dot / 5;
 585 }
 586
 587 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 588
 589 /*
 590  * Returns whether the given set of divisors are valid for a given refclk with
 591  * the given connectors.
 592  */
 593 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 594                                const struct intel_limit *limit,
 595                                const struct dpll *clock)
 596 {
 597         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 598                 INTELPllInvalid("n out of range\n");
 599         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 600                 INTELPllInvalid("p1 out of range\n");
 601         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 602                 INTELPllInvalid("m2 out of range\n");
 603         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 604                 INTELPllInvalid("m1 out of range\n");
 605
 606         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 607             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 608                 if (clock->m1 <= clock->m2)
 609                         INTELPllInvalid("m1 <= m2\n");
 610
 611         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 612             !IS_GEN9_LP(dev_priv)) {
 613                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 614                         INTELPllInvalid("p out of range\n");
 615                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 616                         INTELPllInvalid("m out of range\n");
 617         }
 618
 619         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 620                 INTELPllInvalid("vco out of range\n");
 621         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 622          * connector, etc., rather than just a single range.
 623          */
 624         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 625                 INTELPllInvalid("dot out of range\n");
 626
 627         return true;
 628 }
 629
 630 static int
 631 i9xx_select_p2_div(const struct intel_limit *limit,
 632                    const struct intel_crtc_state *crtc_state,
 633                    int target)
 634 {
 635         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 636
 637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 638                 /*
 639                  * For LVDS just rely on its current settings for dual-channel.
 640                  * We haven't figured out how to reliably set up different
 641                  * single/dual channel state, if we even can.
 642                  */
 643                 if (intel_is_dual_link_lvds(dev_priv))
 644                         return limit->p2.p2_fast;
 645                 else
 646                         return limit->p2.p2_slow;
 647         } else {
 648                 if (target < limit->p2.dot_limit)
 649                         return limit->p2.p2_slow;
 650                 else
 651                         return limit->p2.p2_fast;
 652         }
 653 }
 654
 655 /*
 656  * Returns a set of divisors for the desired target clock with the given
 657  * refclk, or FALSE.  The returned values represent the clock equation:
 658  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 659  *
 660  * Target and reference clocks are specified in kHz.
 661  *
 662  * If match_clock is provided, then best_clock P divider must match the P
 663  * divider from @match_clock used for LVDS downclocking.
 664  */
 665 static bool
 666 i9xx_find_best_dpll(const struct intel_limit *limit,
 667                     struct intel_crtc_state *crtc_state,
 668                     int target, int refclk, struct dpll *match_clock,
 669                     struct dpll *best_clock)
 670 {
 671         struct drm_device *dev = crtc_state->base.crtc->dev;
 672         struct dpll clock;
 673         int err = target;
 674
 675         memset(best_clock, 0, sizeof(*best_clock));
 676
 677         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 678
 679         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 680              clock.m1++) {
 681                 for (clock.m2 = limit->m2.min;
 682                      clock.m2 <= limit->m2.max; clock.m2++) {
 683                         if (clock.m2 >= clock.m1)
 684                                 break;
 685                         for (clock.n = limit->n.min;
 686                              clock.n <= limit->n.max; clock.n++) {
 687                                 for (clock.p1 = limit->p1.min;
 688                                         clock.p1 <= limit->p1.max; clock.p1++) {
 689                                         int this_err;
 690
 691                                         i9xx_calc_dpll_params(refclk, &clock);
 692                                         if (!intel_PLL_is_valid(to_i915(dev),
 693                                                                 limit,
 694                                                                 &clock))
 695                                                 continue;
 696                                         if (match_clock &&
 697                                             clock.p != match_clock->p)
 698                                                 continue;
 699
 700                                         this_err = abs(clock.dot - target);
 701                                         if (this_err < err) {
 702                                                 *best_clock = clock;
 703                                                 err = this_err;
 704                                         }
 705                                 }
 706                         }
 707                 }
 708         }
 709
 710         return (err != target);
 711 }
 712
 713 /*
 714  * Returns a set of divisors for the desired target clock with the given
 715  * refclk, or FALSE.  The returned values represent the clock equation:
 716  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 717  *
 718  * Target and reference clocks are specified in kHz.
 719  *
 720  * If match_clock is provided, then best_clock P divider must match the P
 721  * divider from @match_clock used for LVDS downclocking.
 722  */
 723 static bool
 724 pnv_find_best_dpll(const struct intel_limit *limit,
 725                    struct intel_crtc_state *crtc_state,
 726                    int target, int refclk, struct dpll *match_clock,
 727                    struct dpll *best_clock)
 728 {
 729         struct drm_device *dev = crtc_state->base.crtc->dev;
 730         struct dpll clock;
 731         int err = target;
 732
 733         memset(best_clock, 0, sizeof(*best_clock));
 734
 735         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 736
 737         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 738              clock.m1++) {
 739                 for (clock.m2 = limit->m2.min;
 740                      clock.m2 <= limit->m2.max; clock.m2++) {
 741                         for (clock.n = limit->n.min;
 742                              clock.n <= limit->n.max; clock.n++) {
 743                                 for (clock.p1 = limit->p1.min;
 744                                         clock.p1 <= limit->p1.max; clock.p1++) {
 745                                         int this_err;
 746
 747                                         pnv_calc_dpll_params(refclk, &clock);
 748                                         if (!intel_PLL_is_valid(to_i915(dev),
 749                                                                 limit,
 750                                                                 &clock))
 751                                                 continue;
 752                                         if (match_clock &&
 753                                             clock.p != match_clock->p)
 754                                                 continue;
 755
 756                                         this_err = abs(clock.dot - target);
 757                                         if (this_err < err) {
 758                                                 *best_clock = clock;
 759                                                 err = this_err;
 760                                         }
 761                                 }
 762                         }
 763                 }
 764         }
 765
 766         return (err != target);
 767 }
 768
 769 /*
 770  * Returns a set of divisors for the desired target clock with the given
 771  * refclk, or FALSE.  The returned values represent the clock equation:
 772  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 773  *
 774  * Target and reference clocks are specified in kHz.
 775  *
 776  * If match_clock is provided, then best_clock P divider must match the P
 777  * divider from @match_clock used for LVDS downclocking.
 778  */
 779 static bool
 780 g4x_find_best_dpll(const struct intel_limit *limit,
 781                    struct intel_crtc_state *crtc_state,
 782                    int target, int refclk, struct dpll *match_clock,
 783                    struct dpll *best_clock)
 784 {
 785         struct drm_device *dev = crtc_state->base.crtc->dev;
 786         struct dpll clock;
 787         int max_n;
 788         bool found = false;
 789         /* approximately equals target * 0.00585 */
 790         int err_most = (target >> 8) + (target >> 9);
 791
 792         memset(best_clock, 0, sizeof(*best_clock));
 793
 794         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 795
 796         max_n = limit->n.max;
 797         /* based on hardware requirement, prefer smaller n to precision */
 798         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 799                 /* based on hardware requirement, prefere larger m1,m2 */
 800                 for (clock.m1 = limit->m1.max;
 801                      clock.m1 >= limit->m1.min; clock.m1--) {
 802                         for (clock.m2 = limit->m2.max;
 803                              clock.m2 >= limit->m2.min; clock.m2--) {
 804                                 for (clock.p1 = limit->p1.max;
 805                                      clock.p1 >= limit->p1.min; clock.p1--) {
 806                                         int this_err;
 807
 808                                         i9xx_calc_dpll_params(refclk, &clock);
 809                                         if (!intel_PLL_is_valid(to_i915(dev),
 810                                                                 limit,
 811                                                                 &clock))
 812                                                 continue;
 813
 814                                         this_err = abs(clock.dot - target);
 815                                         if (this_err < err_most) {
 816                                                 *best_clock = clock;
 817                                                 err_most = this_err;
 818                                                 max_n = clock.n;
 819                                                 found = true;
 820                                         }
 821                                 }
 822                         }
 823                 }
 824         }
 825         return found;
 826 }
 827
 828 /*
 829  * Check if the calculated PLL configuration is more optimal compared to the
 830  * best configuration and error found so far. Return the calculated error.
 831  */
 832 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 833                                const struct dpll *calculated_clock,
 834                                const struct dpll *best_clock,
 835                                unsigned int best_error_ppm,
 836                                unsigned int *error_ppm)
 837 {
 838         /*
 839          * For CHV ignore the error and consider only the P value.
 840          * Prefer a bigger P value based on HW requirements.
 841          */
 842         if (IS_CHERRYVIEW(to_i915(dev))) {
 843                 *error_ppm = 0;
 844
 845                 return calculated_clock->p > best_clock->p;
 846         }
 847
 848         if (WARN_ON_ONCE(!target_freq))
 849                 return false;
 850
 851         *error_ppm = div_u64(1000000ULL *
 852                                 abs(target_freq - calculated_clock->dot),
 853                              target_freq);
 854         /*
 855          * Prefer a better P value over a better (smaller) error if the error
 856          * is small. Ensure this preference for future configurations too by
 857          * setting the error to 0.
 858          */
 859         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 860                 *error_ppm = 0;
 861
 862                 return true;
 863         }
 864
 865         return *error_ppm + 10 < best_error_ppm;
 866 }
 867
 868 /*
 869  * Returns a set of divisors for the desired target clock with the given
 870  * refclk, or FALSE.  The returned values represent the clock equation:
 871  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 872  */
 873 static bool
 874 vlv_find_best_dpll(const struct intel_limit *limit,
 875                    struct intel_crtc_state *crtc_state,
 876                    int target, int refclk, struct dpll *match_clock,
 877                    struct dpll *best_clock)
 878 {
 879         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 880         struct drm_device *dev = crtc->base.dev;
 881         struct dpll clock;
 882         unsigned int bestppm = 1000000;
 883         /* min update 19.2 MHz */
 884         int max_n = min(limit->n.max, refclk / 19200);
 885         bool found = false;
 886
 887         target *= 5; /* fast clock */
 888
 889         memset(best_clock, 0, sizeof(*best_clock));
 890
 891         /* based on hardware requirement, prefer smaller n to precision */
 892         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 893                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 894                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 895                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 896                                 clock.p = clock.p1 * clock.p2;
 897                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 898                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 899                                         unsigned int ppm;
 900
 901                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 902                                                                      refclk * clock.m1);
 903
 904                                         vlv_calc_dpll_params(refclk, &clock);
 905
 906                                         if (!intel_PLL_is_valid(to_i915(dev),
 907                                                                 limit,
 908                                                                 &clock))
 909                                                 continue;
 910
 911                                         if (!vlv_PLL_is_optimal(dev, target,
 912                                                                 &clock,
 913                                                                 best_clock,
 914                                                                 bestppm, &ppm))
 915                                                 continue;
 916
 917                                         *best_clock = clock;
 918                                         bestppm = ppm;
 919                                         found = true;
 920                                 }
 921                         }
 922                 }
 923         }
 924
 925         return found;
 926 }
 927
 928 /*
 929  * Returns a set of divisors for the desired target clock with the given
 930  * refclk, or FALSE.  The returned values represent the clock equation:
 931  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 932  */
 933 static bool
 934 chv_find_best_dpll(const struct intel_limit *limit,
 935                    struct intel_crtc_state *crtc_state,
 936                    int target, int refclk, struct dpll *match_clock,
 937                    struct dpll *best_clock)
 938 {
 939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 940         struct drm_device *dev = crtc->base.dev;
 941         unsigned int best_error_ppm;
 942         struct dpll clock;
 943         u64 m2;
 944         int found = false;
 945
 946         memset(best_clock, 0, sizeof(*best_clock));
 947         best_error_ppm = 1000000;
 948
 949         /*
 950          * Based on hardware doc, the n always set to 1, and m1 always
 951          * set to 2.  If requires to support 200Mhz refclk, we need to
 952          * revisit this because n may not 1 anymore.
 953          */
 954         clock.n = 1, clock.m1 = 2;
 955         target *= 5;    /* fast clock */
 956
 957         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 958                 for (clock.p2 = limit->p2.p2_fast;
 959                                 clock.p2 >= limit->p2.p2_slow;
 960                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 961                         unsigned int error_ppm;
 962
 963                         clock.p = clock.p1 * clock.p2;
 964
 965                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
 966                                                    refclk * clock.m1);
 967
 968                         if (m2 > INT_MAX/clock.m1)
 969                                 continue;
 970
 971                         clock.m2 = m2;
 972
 973                         chv_calc_dpll_params(refclk, &clock);
 974
 975                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 976                                 continue;
 977
 978                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 979                                                 best_error_ppm, &error_ppm))
 980                                 continue;
 981
 982                         *best_clock = clock;
 983                         best_error_ppm = error_ppm;
 984                         found = true;
 985                 }
 986         }
 987
 988         return found;
 989 }
 990
 991 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 992                         struct dpll *best_clock)
 993 {
 994         int refclk = 100000;
 995         const struct intel_limit *limit = &intel_limits_bxt;
 996
 997         return chv_find_best_dpll(limit, crtc_state,
 998                                   crtc_state->port_clock, refclk,
 999                                   NULL, best_clock);
1000 }
1001
1002 bool intel_crtc_active(struct intel_crtc *crtc)
1003 {
1004         /* Be paranoid as we can arrive here with only partial
1005          * state retrieved from the hardware during setup.
1006          *
1007          * We can ditch the adjusted_mode.crtc_clock check as soon
1008          * as Haswell has gained clock readout/fastboot support.
1009          *
1010          * We can ditch the crtc->primary->state->fb check as soon as we can
1011          * properly reconstruct framebuffers.
1012          *
1013          * FIXME: The intel_crtc->active here should be switched to
1014          * crtc->state->active once we have proper CRTC states wired up
1015          * for atomic.
1016          */
1017         return crtc->active && crtc->base.primary->state->fb &&
1018                 crtc->config->base.adjusted_mode.crtc_clock;
1019 }
1020
1021 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1022                                              enum pipe pipe)
1023 {
1024         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1025
1026         return crtc->config->cpu_transcoder;
1027 }
1028
1029 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1030                                     enum pipe pipe)
1031 {
1032         i915_reg_t reg = PIPEDSL(pipe);
1033         u32 line1, line2;
1034         u32 line_mask;
1035
1036         if (IS_GEN(dev_priv, 2))
1037                 line_mask = DSL_LINEMASK_GEN2;
1038         else
1039                 line_mask = DSL_LINEMASK_GEN3;
1040
1041         line1 = I915_READ(reg) & line_mask;
1042         msleep(5);
1043         line2 = I915_READ(reg) & line_mask;
1044
1045         return line1 != line2;
1046 }
1047
1048 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1049 {
1050         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1051         enum pipe pipe = crtc->pipe;
1052
1053         /* Wait for the display line to settle/start moving */
1054         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1055                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1056                           pipe_name(pipe), onoff(state));
1057 }
1058
1059 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1060 {
1061         wait_for_pipe_scanline_moving(crtc, false);
1062 }
1063
1064 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1065 {
1066         wait_for_pipe_scanline_moving(crtc, true);
1067 }
1068
1069 static void
1070 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1071 {
1072         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1073         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1074
1075         if (INTEL_GEN(dev_priv) >= 4) {
1076                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1077                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1078
1079                 /* Wait for the Pipe State to go off */
1080                 if (intel_de_wait_for_clear(dev_priv, reg,
1081                                             I965_PIPECONF_ACTIVE, 100))
1082                         WARN(1, "pipe_off wait timed out\n");
1083         } else {
1084                 intel_wait_for_pipe_scanline_stopped(crtc);
1085         }
1086 }
1087
1088 /* Only for pre-ILK configs */
1089 void assert_pll(struct drm_i915_private *dev_priv,
1090                 enum pipe pipe, bool state)
1091 {
1092         u32 val;
1093         bool cur_state;
1094
1095         val = I915_READ(DPLL(pipe));
1096         cur_state = !!(val & DPLL_VCO_ENABLE);
1097         I915_STATE_WARN(cur_state != state,
1098              "PLL state assertion failure (expected %s, current %s)\n",
1099                         onoff(state), onoff(cur_state));
1100 }
1101
1102 /* XXX: the dsi pll is shared between MIPI DSI ports */
1103 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1104 {
1105         u32 val;
1106         bool cur_state;
1107
1108         vlv_cck_get(dev_priv);
1109         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1110         vlv_cck_put(dev_priv);
1111
1112         cur_state = val & DSI_PLL_VCO_EN;
1113         I915_STATE_WARN(cur_state != state,
1114              "DSI PLL state assertion failure (expected %s, current %s)\n",
1115                         onoff(state), onoff(cur_state));
1116 }
1117
1118 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1119                           enum pipe pipe, bool state)
1120 {
1121         bool cur_state;
1122         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1123                                                                       pipe);
1124
1125         if (HAS_DDI(dev_priv)) {
1126                 /* DDI does not have a specific FDI_TX register */
1127                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1128                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1129         } else {
1130                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1131                 cur_state = !!(val & FDI_TX_ENABLE);
1132         }
1133         I915_STATE_WARN(cur_state != state,
1134              "FDI TX state assertion failure (expected %s, current %s)\n",
1135                         onoff(state), onoff(cur_state));
1136 }
1137 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1138 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1139
1140 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1141                           enum pipe pipe, bool state)
1142 {
1143         u32 val;
1144         bool cur_state;
1145
1146         val = I915_READ(FDI_RX_CTL(pipe));
1147         cur_state = !!(val & FDI_RX_ENABLE);
1148         I915_STATE_WARN(cur_state != state,
1149              "FDI RX state assertion failure (expected %s, current %s)\n",
1150                         onoff(state), onoff(cur_state));
1151 }
1152 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1153 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1154
1155 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1156                                       enum pipe pipe)
1157 {
1158         u32 val;
1159
1160         /* ILK FDI PLL is always enabled */
1161         if (IS_GEN(dev_priv, 5))
1162                 return;
1163
1164         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1165         if (HAS_DDI(dev_priv))
1166                 return;
1167
1168         val = I915_READ(FDI_TX_CTL(pipe));
1169         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1170 }
1171
1172 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1173                        enum pipe pipe, bool state)
1174 {
1175         u32 val;
1176         bool cur_state;
1177
1178         val = I915_READ(FDI_RX_CTL(pipe));
1179         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1180         I915_STATE_WARN(cur_state != state,
1181              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1182                         onoff(state), onoff(cur_state));
1183 }
1184
1185 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1186 {
1187         i915_reg_t pp_reg;
1188         u32 val;
1189         enum pipe panel_pipe = INVALID_PIPE;
1190         bool locked = true;
1191
1192         if (WARN_ON(HAS_DDI(dev_priv)))
1193                 return;
1194
1195         if (HAS_PCH_SPLIT(dev_priv)) {
1196                 u32 port_sel;
1197
1198                 pp_reg = PP_CONTROL(0);
1199                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1200
1201                 switch (port_sel) {
1202                 case PANEL_PORT_SELECT_LVDS:
1203                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1204                         break;
1205                 case PANEL_PORT_SELECT_DPA:
1206                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1207                         break;
1208                 case PANEL_PORT_SELECT_DPC:
1209                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1210                         break;
1211                 case PANEL_PORT_SELECT_DPD:
1212                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1213                         break;
1214                 default:
1215                         MISSING_CASE(port_sel);
1216                         break;
1217                 }
1218         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1219                 /* presumably write lock depends on pipe, not port select */
1220                 pp_reg = PP_CONTROL(pipe);
1221                 panel_pipe = pipe;
1222         } else {
1223                 u32 port_sel;
1224
1225                 pp_reg = PP_CONTROL(0);
1226                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1227
1228                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1229                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1230         }
1231
1232         val = I915_READ(pp_reg);
1233         if (!(val & PANEL_POWER_ON) ||
1234             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1235                 locked = false;
1236
1237         I915_STATE_WARN(panel_pipe == pipe && locked,
1238              "panel assertion failure, pipe %c regs locked\n",
1239              pipe_name(pipe));
1240 }
1241
1242 void assert_pipe(struct drm_i915_private *dev_priv,
1243                  enum pipe pipe, bool state)
1244 {
1245         bool cur_state;
1246         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1247                                                                       pipe);
1248         enum intel_display_power_domain power_domain;
1249         intel_wakeref_t wakeref;
1250
1251         /* we keep both pipes enabled on 830 */
1252         if (IS_I830(dev_priv))
1253                 state = true;
1254
1255         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1256         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1257         if (wakeref) {
1258                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1259                 cur_state = !!(val & PIPECONF_ENABLE);
1260
1261                 intel_display_power_put(dev_priv, power_domain, wakeref);
1262         } else {
1263                 cur_state = false;
1264         }
1265
1266         I915_STATE_WARN(cur_state != state,
1267              "pipe %c assertion failure (expected %s, current %s)\n",
1268                         pipe_name(pipe), onoff(state), onoff(cur_state));
1269 }
1270
1271 static void assert_plane(struct intel_plane *plane, bool state)
1272 {
1273         enum pipe pipe;
1274         bool cur_state;
1275
1276         cur_state = plane->get_hw_state(plane, &pipe);
1277
1278         I915_STATE_WARN(cur_state != state,
1279                         "%s assertion failure (expected %s, current %s)\n",
1280                         plane->base.name, onoff(state), onoff(cur_state));
1281 }
1282
1283 #define assert_plane_enabled(p) assert_plane(p, true)
1284 #define assert_plane_disabled(p) assert_plane(p, false)
1285
1286 static void assert_planes_disabled(struct intel_crtc *crtc)
1287 {
1288         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1289         struct intel_plane *plane;
1290
1291         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1292                 assert_plane_disabled(plane);
1293 }
1294
1295 static void assert_vblank_disabled(struct drm_crtc *crtc)
1296 {
1297         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1298                 drm_crtc_vblank_put(crtc);
1299 }
1300
1301 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1302                                     enum pipe pipe)
1303 {
1304         u32 val;
1305         bool enabled;
1306
1307         val = I915_READ(PCH_TRANSCONF(pipe));
1308         enabled = !!(val & TRANS_ENABLE);
1309         I915_STATE_WARN(enabled,
1310              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1311              pipe_name(pipe));
1312 }
1313
1314 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1315                                    enum pipe pipe, enum port port,
1316                                    i915_reg_t dp_reg)
1317 {
1318         enum pipe port_pipe;
1319         bool state;
1320
1321         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1322
1323         I915_STATE_WARN(state && port_pipe == pipe,
1324                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1325                         port_name(port), pipe_name(pipe));
1326
1327         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1328                         "IBX PCH DP %c still using transcoder B\n",
1329                         port_name(port));
1330 }
1331
1332 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1333                                      enum pipe pipe, enum port port,
1334                                      i915_reg_t hdmi_reg)
1335 {
1336         enum pipe port_pipe;
1337         bool state;
1338
1339         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1340
1341         I915_STATE_WARN(state && port_pipe == pipe,
1342                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1343                         port_name(port), pipe_name(pipe));
1344
1345         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1346                         "IBX PCH HDMI %c still using transcoder B\n",
1347                         port_name(port));
1348 }
1349
1350 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1351                                       enum pipe pipe)
1352 {
1353         enum pipe port_pipe;
1354
1355         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1356         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1357         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1358
1359         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1360                         port_pipe == pipe,
1361                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1362                         pipe_name(pipe));
1363
1364         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1365                         port_pipe == pipe,
1366                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1367                         pipe_name(pipe));
1368
1369         /* PCH SDVOB multiplex with HDMIB */
1370         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1371         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1372         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1373 }
1374
1375 static void _vlv_enable_pll(struct intel_crtc *crtc,
1376                             const struct intel_crtc_state *pipe_config)
1377 {
1378         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1379         enum pipe pipe = crtc->pipe;
1380
1381         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1382         POSTING_READ(DPLL(pipe));
1383         udelay(150);
1384
1385         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1386                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1387 }
1388
1389 static void vlv_enable_pll(struct intel_crtc *crtc,
1390                            const struct intel_crtc_state *pipe_config)
1391 {
1392         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1393         enum pipe pipe = crtc->pipe;
1394
1395         assert_pipe_disabled(dev_priv, pipe);
1396
1397         /* PLL is protected by panel, make sure we can write it */
1398         assert_panel_unlocked(dev_priv, pipe);
1399
1400         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1401                 _vlv_enable_pll(crtc, pipe_config);
1402
1403         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1404         POSTING_READ(DPLL_MD(pipe));
1405 }
1406
1407
1408 static void _chv_enable_pll(struct intel_crtc *crtc,
1409                             const struct intel_crtc_state *pipe_config)
1410 {
1411         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1412         enum pipe pipe = crtc->pipe;
1413         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1414         u32 tmp;
1415
1416         vlv_dpio_get(dev_priv);
1417
1418         /* Enable back the 10bit clock to display controller */
1419         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1420         tmp |= DPIO_DCLKP_EN;
1421         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1422
1423         vlv_dpio_put(dev_priv);
1424
1425         /*
1426          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1427          */
1428         udelay(1);
1429
1430         /* Enable PLL */
1431         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1432
1433         /* Check PLL is locked */
1434         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1435                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1436 }
1437
1438 static void chv_enable_pll(struct intel_crtc *crtc,
1439                            const struct intel_crtc_state *pipe_config)
1440 {
1441         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1442         enum pipe pipe = crtc->pipe;
1443
1444         assert_pipe_disabled(dev_priv, pipe);
1445
1446         /* PLL is protected by panel, make sure we can write it */
1447         assert_panel_unlocked(dev_priv, pipe);
1448
1449         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1450                 _chv_enable_pll(crtc, pipe_config);
1451
1452         if (pipe != PIPE_A) {
1453                 /*
1454                  * WaPixelRepeatModeFixForC0:chv
1455                  *
1456                  * DPLLCMD is AWOL. Use chicken bits to propagate
1457                  * the value from DPLLBMD to either pipe B or C.
1458                  */
1459                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1460                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1461                 I915_WRITE(CBR4_VLV, 0);
1462                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1463
1464                 /*
1465                  * DPLLB VGA mode also seems to cause problems.
1466                  * We should always have it disabled.
1467                  */
1468                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1469         } else {
1470                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1471                 POSTING_READ(DPLL_MD(pipe));
1472         }
1473 }
1474
1475 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1476 {
1477         if (IS_I830(dev_priv))
1478                 return false;
1479
1480         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1481 }
1482
1483 static void i9xx_enable_pll(struct intel_crtc *crtc,
1484                             const struct intel_crtc_state *crtc_state)
1485 {
1486         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1487         i915_reg_t reg = DPLL(crtc->pipe);
1488         u32 dpll = crtc_state->dpll_hw_state.dpll;
1489         int i;
1490
1491         assert_pipe_disabled(dev_priv, crtc->pipe);
1492
1493         /* PLL is protected by panel, make sure we can write it */
1494         if (i9xx_has_pps(dev_priv))
1495                 assert_panel_unlocked(dev_priv, crtc->pipe);
1496
1497         /*
1498          * Apparently we need to have VGA mode enabled prior to changing
1499          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1500          * dividers, even though the register value does change.
1501          */
1502         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1503         I915_WRITE(reg, dpll);
1504
1505         /* Wait for the clocks to stabilize. */
1506         POSTING_READ(reg);
1507         udelay(150);
1508
1509         if (INTEL_GEN(dev_priv) >= 4) {
1510                 I915_WRITE(DPLL_MD(crtc->pipe),
1511                            crtc_state->dpll_hw_state.dpll_md);
1512         } else {
1513                 /* The pixel multiplier can only be updated once the
1514                  * DPLL is enabled and the clocks are stable.
1515                  *
1516                  * So write it again.
1517                  */
1518                 I915_WRITE(reg, dpll);
1519         }
1520
1521         /* We do this three times for luck */
1522         for (i = 0; i < 3; i++) {
1523                 I915_WRITE(reg, dpll);
1524                 POSTING_READ(reg);
1525                 udelay(150); /* wait for warmup */
1526         }
1527 }
1528
1529 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1530 {
1531         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1532         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1533         enum pipe pipe = crtc->pipe;
1534
1535         /* Don't disable pipe or pipe PLLs if needed */
1536         if (IS_I830(dev_priv))
1537                 return;
1538
1539         /* Make sure the pipe isn't still relying on us */
1540         assert_pipe_disabled(dev_priv, pipe);
1541
1542         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1543         POSTING_READ(DPLL(pipe));
1544 }
1545
1546 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1547 {
1548         u32 val;
1549
1550         /* Make sure the pipe isn't still relying on us */
1551         assert_pipe_disabled(dev_priv, pipe);
1552
1553         val = DPLL_INTEGRATED_REF_CLK_VLV |
1554                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1555         if (pipe != PIPE_A)
1556                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1557
1558         I915_WRITE(DPLL(pipe), val);
1559         POSTING_READ(DPLL(pipe));
1560 }
1561
1562 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1563 {
1564         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1565         u32 val;
1566
1567         /* Make sure the pipe isn't still relying on us */
1568         assert_pipe_disabled(dev_priv, pipe);
1569
1570         val = DPLL_SSC_REF_CLK_CHV |
1571                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1572         if (pipe != PIPE_A)
1573                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1574
1575         I915_WRITE(DPLL(pipe), val);
1576         POSTING_READ(DPLL(pipe));
1577
1578         vlv_dpio_get(dev_priv);
1579
1580         /* Disable 10bit clock to display controller */
1581         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1582         val &= ~DPIO_DCLKP_EN;
1583         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1584
1585         vlv_dpio_put(dev_priv);
1586 }
1587
1588 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1589                          struct intel_digital_port *dport,
1590                          unsigned int expected_mask)
1591 {
1592         u32 port_mask;
1593         i915_reg_t dpll_reg;
1594
1595         switch (dport->base.port) {
1596         case PORT_B:
1597                 port_mask = DPLL_PORTB_READY_MASK;
1598                 dpll_reg = DPLL(0);
1599                 break;
1600         case PORT_C:
1601                 port_mask = DPLL_PORTC_READY_MASK;
1602                 dpll_reg = DPLL(0);
1603                 expected_mask <<= 4;
1604                 break;
1605         case PORT_D:
1606                 port_mask = DPLL_PORTD_READY_MASK;
1607                 dpll_reg = DPIO_PHY_STATUS;
1608                 break;
1609         default:
1610                 BUG();
1611         }
1612
1613         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1614                                        port_mask, expected_mask, 1000))
1615                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1616                      port_name(dport->base.port),
1617                      I915_READ(dpll_reg) & port_mask, expected_mask);
1618 }
1619
1620 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1621 {
1622         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1623         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1624         enum pipe pipe = crtc->pipe;
1625         i915_reg_t reg;
1626         u32 val, pipeconf_val;
1627
1628         /* Make sure PCH DPLL is enabled */
1629         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1630
1631         /* FDI must be feeding us bits for PCH ports */
1632         assert_fdi_tx_enabled(dev_priv, pipe);
1633         assert_fdi_rx_enabled(dev_priv, pipe);
1634
1635         if (HAS_PCH_CPT(dev_priv)) {
1636                 /* Workaround: Set the timing override bit before enabling the
1637                  * pch transcoder. */
1638                 reg = TRANS_CHICKEN2(pipe);
1639                 val = I915_READ(reg);
1640                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1641                 I915_WRITE(reg, val);
1642         }
1643
1644         reg = PCH_TRANSCONF(pipe);
1645         val = I915_READ(reg);
1646         pipeconf_val = I915_READ(PIPECONF(pipe));
1647
1648         if (HAS_PCH_IBX(dev_priv)) {
1649                 /*
1650                  * Make the BPC in transcoder be consistent with
1651                  * that in pipeconf reg. For HDMI we must use 8bpc
1652                  * here for both 8bpc and 12bpc.
1653                  */
1654                 val &= ~PIPECONF_BPC_MASK;
1655                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1656                         val |= PIPECONF_8BPC;
1657                 else
1658                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1659         }
1660
1661         val &= ~TRANS_INTERLACE_MASK;
1662         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1663                 if (HAS_PCH_IBX(dev_priv) &&
1664                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1665                         val |= TRANS_LEGACY_INTERLACED_ILK;
1666                 else
1667                         val |= TRANS_INTERLACED;
1668         } else {
1669                 val |= TRANS_PROGRESSIVE;
1670         }
1671
1672         I915_WRITE(reg, val | TRANS_ENABLE);
1673         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1674                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1675 }
1676
1677 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1678                                       enum transcoder cpu_transcoder)
1679 {
1680         u32 val, pipeconf_val;
1681
1682         /* FDI must be feeding us bits for PCH ports */
1683         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1684         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1685
1686         /* Workaround: set timing override bit. */
1687         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1688         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1689         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1690
1691         val = TRANS_ENABLE;
1692         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1693
1694         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1695             PIPECONF_INTERLACED_ILK)
1696                 val |= TRANS_INTERLACED;
1697         else
1698                 val |= TRANS_PROGRESSIVE;
1699
1700         I915_WRITE(LPT_TRANSCONF, val);
1701         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1702                                   TRANS_STATE_ENABLE, 100))
1703                 DRM_ERROR("Failed to enable PCH transcoder\n");
1704 }
1705
1706 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1707                                             enum pipe pipe)
1708 {
1709         i915_reg_t reg;
1710         u32 val;
1711
1712         /* FDI relies on the transcoder */
1713         assert_fdi_tx_disabled(dev_priv, pipe);
1714         assert_fdi_rx_disabled(dev_priv, pipe);
1715
1716         /* Ports must be off as well */
1717         assert_pch_ports_disabled(dev_priv, pipe);
1718
1719         reg = PCH_TRANSCONF(pipe);
1720         val = I915_READ(reg);
1721         val &= ~TRANS_ENABLE;
1722         I915_WRITE(reg, val);
1723         /* wait for PCH transcoder off, transcoder state */
1724         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1725                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1726
1727         if (HAS_PCH_CPT(dev_priv)) {
1728                 /* Workaround: Clear the timing override chicken bit again. */
1729                 reg = TRANS_CHICKEN2(pipe);
1730                 val = I915_READ(reg);
1731                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1732                 I915_WRITE(reg, val);
1733         }
1734 }
1735
1736 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1737 {
1738         u32 val;
1739
1740         val = I915_READ(LPT_TRANSCONF);
1741         val &= ~TRANS_ENABLE;
1742         I915_WRITE(LPT_TRANSCONF, val);
1743         /* wait for PCH transcoder off, transcoder state */
1744         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1745                                     TRANS_STATE_ENABLE, 50))
1746                 DRM_ERROR("Failed to disable PCH transcoder\n");
1747
1748         /* Workaround: clear timing override bit. */
1749         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1750         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1751         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1752 }
1753
1754 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1755 {
1756         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1757
1758         if (HAS_PCH_LPT(dev_priv))
1759                 return PIPE_A;
1760         else
1761                 return crtc->pipe;
1762 }
1763
1764 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1765 {
1766         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1767
1768         /*
1769          * On i965gm the hardware frame counter reads
1770          * zero when the TV encoder is enabled :(
1771          */
1772         if (IS_I965GM(dev_priv) &&
1773             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1774                 return 0;
1775
1776         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1777                 return 0xffffffff; /* full 32 bit counter */
1778         else if (INTEL_GEN(dev_priv) >= 3)
1779                 return 0xffffff; /* only 24 bits of frame count */
1780         else
1781                 return 0; /* Gen2 doesn't have a hardware frame counter */
1782 }
1783
1784 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1785 {
1786         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1787
1788         drm_crtc_set_max_vblank_count(&crtc->base,
1789                                       intel_crtc_max_vblank_count(crtc_state));
1790         drm_crtc_vblank_on(&crtc->base);
1791 }
1792
1793 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1794 {
1795         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1797         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1798         enum pipe pipe = crtc->pipe;
1799         i915_reg_t reg;
1800         u32 val;
1801
1802         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1803
1804         assert_planes_disabled(crtc);
1805
1806         /*
1807          * A pipe without a PLL won't actually be able to drive bits from
1808          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1809          * need the check.
1810          */
1811         if (HAS_GMCH(dev_priv)) {
1812                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1813                         assert_dsi_pll_enabled(dev_priv);
1814                 else
1815                         assert_pll_enabled(dev_priv, pipe);
1816         } else {
1817                 if (new_crtc_state->has_pch_encoder) {
1818                         /* if driving the PCH, we need FDI enabled */
1819                         assert_fdi_rx_pll_enabled(dev_priv,
1820                                                   intel_crtc_pch_transcoder(crtc));
1821                         assert_fdi_tx_pll_enabled(dev_priv,
1822                                                   (enum pipe) cpu_transcoder);
1823                 }
1824                 /* FIXME: assert CPU port conditions for SNB+ */
1825         }
1826
1827         trace_intel_pipe_enable(crtc);
1828
1829         reg = PIPECONF(cpu_transcoder);
1830         val = I915_READ(reg);
1831         if (val & PIPECONF_ENABLE) {
1832                 /* we keep both pipes enabled on 830 */
1833                 WARN_ON(!IS_I830(dev_priv));
1834                 return;
1835         }
1836
1837         I915_WRITE(reg, val | PIPECONF_ENABLE);
1838         POSTING_READ(reg);
1839
1840         /*
1841          * Until the pipe starts PIPEDSL reads will return a stale value,
1842          * which causes an apparent vblank timestamp jump when PIPEDSL
1843          * resets to its proper value. That also messes up the frame count
1844          * when it's derived from the timestamps. So let's wait for the
1845          * pipe to start properly before we call drm_crtc_vblank_on()
1846          */
1847         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1848                 intel_wait_for_pipe_scanline_moving(crtc);
1849 }
1850
1851 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1852 {
1853         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1854         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1855         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1856         enum pipe pipe = crtc->pipe;
1857         i915_reg_t reg;
1858         u32 val;
1859
1860         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1861
1862         /*
1863          * Make sure planes won't keep trying to pump pixels to us,
1864          * or we might hang the display.
1865          */
1866         assert_planes_disabled(crtc);
1867
1868         trace_intel_pipe_disable(crtc);
1869
1870         reg = PIPECONF(cpu_transcoder);
1871         val = I915_READ(reg);
1872         if ((val & PIPECONF_ENABLE) == 0)
1873                 return;
1874
1875         /*
1876          * Double wide has implications for planes
1877          * so best keep it disabled when not needed.
1878          */
1879         if (old_crtc_state->double_wide)
1880                 val &= ~PIPECONF_DOUBLE_WIDE;
1881
1882         /* Don't disable pipe or pipe PLLs if needed */
1883         if (!IS_I830(dev_priv))
1884                 val &= ~PIPECONF_ENABLE;
1885
1886         I915_WRITE(reg, val);
1887         if ((val & PIPECONF_ENABLE) == 0)
1888                 intel_wait_for_pipe_off(old_crtc_state);
1889 }
1890
1891 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1892 {
1893         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1894 }
1895
1896 static unsigned int
1897 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1898 {
1899         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1900         unsigned int cpp = fb->format->cpp[color_plane];
1901
1902         switch (fb->modifier) {
1903         case DRM_FORMAT_MOD_LINEAR:
1904                 return intel_tile_size(dev_priv);
1905         case I915_FORMAT_MOD_X_TILED:
1906                 if (IS_GEN(dev_priv, 2))
1907                         return 128;
1908                 else
1909                         return 512;
1910         case I915_FORMAT_MOD_Y_TILED_CCS:
1911                 if (color_plane == 1)
1912                         return 128;
1913                 /* fall through */
1914         case I915_FORMAT_MOD_Y_TILED:
1915                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1916                         return 128;
1917                 else
1918                         return 512;
1919         case I915_FORMAT_MOD_Yf_TILED_CCS:
1920                 if (color_plane == 1)
1921                         return 128;
1922                 /* fall through */
1923         case I915_FORMAT_MOD_Yf_TILED:
1924                 switch (cpp) {
1925                 case 1:
1926                         return 64;
1927                 case 2:
1928                 case 4:
1929                         return 128;
1930                 case 8:
1931                 case 16:
1932                         return 256;
1933                 default:
1934                         MISSING_CASE(cpp);
1935                         return cpp;
1936                 }
1937                 break;
1938         default:
1939                 MISSING_CASE(fb->modifier);
1940                 return cpp;
1941         }
1942 }
1943
1944 static unsigned int
1945 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1946 {
1947         return intel_tile_size(to_i915(fb->dev)) /
1948                 intel_tile_width_bytes(fb, color_plane);
1949 }
1950
1951 /* Return the tile dimensions in pixel units */
1952 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1953                             unsigned int *tile_width,
1954                             unsigned int *tile_height)
1955 {
1956         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1957         unsigned int cpp = fb->format->cpp[color_plane];
1958
1959         *tile_width = tile_width_bytes / cpp;
1960         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1961 }
1962
1963 unsigned int
1964 intel_fb_align_height(const struct drm_framebuffer *fb,
1965                       int color_plane, unsigned int height)
1966 {
1967         unsigned int tile_height = intel_tile_height(fb, color_plane);
1968
1969         return ALIGN(height, tile_height);
1970 }
1971
1972 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1973 {
1974         unsigned int size = 0;
1975         int i;
1976
1977         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1978                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1979
1980         return size;
1981 }
1982
1983 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
1984 {
1985         unsigned int size = 0;
1986         int i;
1987
1988         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1989                 size += rem_info->plane[i].width * rem_info->plane[i].height;
1990
1991         return size;
1992 }
1993
1994 static void
1995 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
1996                         const struct drm_framebuffer *fb,
1997                         unsigned int rotation)
1998 {
1999         view->type = I915_GGTT_VIEW_NORMAL;
2000         if (drm_rotation_90_or_270(rotation)) {
2001                 view->type = I915_GGTT_VIEW_ROTATED;
2002                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2003         }
2004 }
2005
2006 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2007 {
2008         if (IS_I830(dev_priv))
2009                 return 16 * 1024;
2010         else if (IS_I85X(dev_priv))
2011                 return 256;
2012         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2013                 return 32;
2014         else
2015                 return 4 * 1024;
2016 }
2017
2018 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2019 {
2020         if (INTEL_GEN(dev_priv) >= 9)
2021                 return 256 * 1024;
2022         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2023                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2024                 return 128 * 1024;
2025         else if (INTEL_GEN(dev_priv) >= 4)
2026                 return 4 * 1024;
2027         else
2028                 return 0;
2029 }
2030
2031 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2032                                          int color_plane)
2033 {
2034         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2035
2036         /* AUX_DIST needs only 4K alignment */
2037         if (color_plane == 1)
2038                 return 4096;
2039
2040         switch (fb->modifier) {
2041         case DRM_FORMAT_MOD_LINEAR:
2042                 return intel_linear_alignment(dev_priv);
2043         case I915_FORMAT_MOD_X_TILED:
2044                 if (INTEL_GEN(dev_priv) >= 9)
2045                         return 256 * 1024;
2046                 return 0;
2047         case I915_FORMAT_MOD_Y_TILED_CCS:
2048         case I915_FORMAT_MOD_Yf_TILED_CCS:
2049         case I915_FORMAT_MOD_Y_TILED:
2050         case I915_FORMAT_MOD_Yf_TILED:
2051                 return 1 * 1024 * 1024;
2052         default:
2053                 MISSING_CASE(fb->modifier);
2054                 return 0;
2055         }
2056 }
2057
2058 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2059 {
2060         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2061         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2062
2063         return INTEL_GEN(dev_priv) < 4 ||
2064                 (plane->has_fbc &&
2065                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2066 }
2067
2068 struct i915_vma *
2069 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2070                            const struct i915_ggtt_view *view,
2071                            bool uses_fence,
2072                            unsigned long *out_flags)
2073 {
2074         struct drm_device *dev = fb->dev;
2075         struct drm_i915_private *dev_priv = to_i915(dev);
2076         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2077         intel_wakeref_t wakeref;
2078         struct i915_vma *vma;
2079         unsigned int pinctl;
2080         u32 alignment;
2081
2082         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2083
2084         alignment = intel_surf_alignment(fb, 0);
2085
2086         /* Note that the w/a also requires 64 PTE of padding following the
2087          * bo. We currently fill all unused PTE with the shadow page and so
2088          * we should always have valid PTE following the scanout preventing
2089          * the VT-d warning.
2090          */
2091         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2092                 alignment = 256 * 1024;
2093
2094         /*
2095          * Global gtt pte registers are special registers which actually forward
2096          * writes to a chunk of system memory. Which means that there is no risk
2097          * that the register values disappear as soon as we call
2098          * intel_runtime_pm_put(), so it is correct to wrap only the
2099          * pin/unpin/fence and not more.
2100          */
2101         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2102         i915_gem_object_lock(obj);
2103
2104         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2105
2106         pinctl = 0;
2107
2108         /* Valleyview is definitely limited to scanning out the first
2109          * 512MiB. Lets presume this behaviour was inherited from the
2110          * g4x display engine and that all earlier gen are similarly
2111          * limited. Testing suggests that it is a little more
2112          * complicated than this. For example, Cherryview appears quite
2113          * happy to scanout from anywhere within its global aperture.
2114          */
2115         if (HAS_GMCH(dev_priv))
2116                 pinctl |= PIN_MAPPABLE;
2117
2118         vma = i915_gem_object_pin_to_display_plane(obj,
2119                                                    alignment, view, pinctl);
2120         if (IS_ERR(vma))
2121                 goto err;
2122
2123         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2124                 int ret;
2125
2126                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2127                  * fence, whereas 965+ only requires a fence if using
2128                  * framebuffer compression.  For simplicity, we always, when
2129                  * possible, install a fence as the cost is not that onerous.
2130                  *
2131                  * If we fail to fence the tiled scanout, then either the
2132                  * modeset will reject the change (which is highly unlikely as
2133                  * the affected systems, all but one, do not have unmappable
2134                  * space) or we will not be able to enable full powersaving
2135                  * techniques (also likely not to apply due to various limits
2136                  * FBC and the like impose on the size of the buffer, which
2137                  * presumably we violated anyway with this unmappable buffer).
2138                  * Anyway, it is presumably better to stumble onwards with
2139                  * something and try to run the system in a "less than optimal"
2140                  * mode that matches the user configuration.
2141                  */
2142                 ret = i915_vma_pin_fence(vma);
2143                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2144                         i915_gem_object_unpin_from_display_plane(vma);
2145                         vma = ERR_PTR(ret);
2146                         goto err;
2147                 }
2148
2149                 if (ret == 0 && vma->fence)
2150                         *out_flags |= PLANE_HAS_FENCE;
2151         }
2152
2153         i915_vma_get(vma);
2154 err:
2155         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2156
2157         i915_gem_object_unlock(obj);
2158         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2159         return vma;
2160 }
2161
2162 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2163 {
2164         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2165
2166         i915_gem_object_lock(vma->obj);
2167         if (flags & PLANE_HAS_FENCE)
2168                 i915_vma_unpin_fence(vma);
2169         i915_gem_object_unpin_from_display_plane(vma);
2170         i915_gem_object_unlock(vma->obj);
2171
2172         i915_vma_put(vma);
2173 }
2174
2175 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2176                           unsigned int rotation)
2177 {
2178         if (drm_rotation_90_or_270(rotation))
2179                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2180         else
2181                 return fb->pitches[color_plane];
2182 }
2183
2184 /*
2185  * Convert the x/y offsets into a linear offset.
2186  * Only valid with 0/180 degree rotation, which is fine since linear
2187  * offset is only used with linear buffers on pre-hsw and tiled buffers
2188  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2189  */
2190 u32 intel_fb_xy_to_linear(int x, int y,
2191                           const struct intel_plane_state *state,
2192                           int color_plane)
2193 {
2194         const struct drm_framebuffer *fb = state->base.fb;
2195         unsigned int cpp = fb->format->cpp[color_plane];
2196         unsigned int pitch = state->color_plane[color_plane].stride;
2197
2198         return y * pitch + x * cpp;
2199 }
2200
2201 /*
2202  * Add the x/y offsets derived from fb->offsets[] to the user
2203  * specified plane src x/y offsets. The resulting x/y offsets
2204  * specify the start of scanout from the beginning of the gtt mapping.
2205  */
2206 void intel_add_fb_offsets(int *x, int *y,
2207                           const struct intel_plane_state *state,
2208                           int color_plane)
2209
2210 {
2211         *x += state->color_plane[color_plane].x;
2212         *y += state->color_plane[color_plane].y;
2213 }
2214
2215 static u32 intel_adjust_tile_offset(int *x, int *y,
2216                                     unsigned int tile_width,
2217                                     unsigned int tile_height,
2218                                     unsigned int tile_size,
2219                                     unsigned int pitch_tiles,
2220                                     u32 old_offset,
2221                                     u32 new_offset)
2222 {
2223         unsigned int pitch_pixels = pitch_tiles * tile_width;
2224         unsigned int tiles;
2225
2226         WARN_ON(old_offset & (tile_size - 1));
2227         WARN_ON(new_offset & (tile_size - 1));
2228         WARN_ON(new_offset > old_offset);
2229
2230         tiles = (old_offset - new_offset) / tile_size;
2231
2232         *y += tiles / pitch_tiles * tile_height;
2233         *x += tiles % pitch_tiles * tile_width;
2234
2235         /* minimize x in case it got needlessly big */
2236         *y += *x / pitch_pixels * tile_height;
2237         *x %= pitch_pixels;
2238
2239         return new_offset;
2240 }
2241
2242 static bool is_surface_linear(u64 modifier, int color_plane)
2243 {
2244         return modifier == DRM_FORMAT_MOD_LINEAR;
2245 }
2246
2247 static u32 intel_adjust_aligned_offset(int *x, int *y,
2248                                        const struct drm_framebuffer *fb,
2249                                        int color_plane,
2250                                        unsigned int rotation,
2251                                        unsigned int pitch,
2252                                        u32 old_offset, u32 new_offset)
2253 {
2254         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2255         unsigned int cpp = fb->format->cpp[color_plane];
2256
2257         WARN_ON(new_offset > old_offset);
2258
2259         if (!is_surface_linear(fb->modifier, color_plane)) {
2260                 unsigned int tile_size, tile_width, tile_height;
2261                 unsigned int pitch_tiles;
2262
2263                 tile_size = intel_tile_size(dev_priv);
2264                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2265
2266                 if (drm_rotation_90_or_270(rotation)) {
2267                         pitch_tiles = pitch / tile_height;
2268                         swap(tile_width, tile_height);
2269                 } else {
2270                         pitch_tiles = pitch / (tile_width * cpp);
2271                 }
2272
2273                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2274                                          tile_size, pitch_tiles,
2275                                          old_offset, new_offset);
2276         } else {
2277                 old_offset += *y * pitch + *x * cpp;
2278
2279                 *y = (old_offset - new_offset) / pitch;
2280                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2281         }
2282
2283         return new_offset;
2284 }
2285
2286 /*
2287  * Adjust the tile offset by moving the difference into
2288  * the x/y offsets.
2289  */
2290 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2291                                              const struct intel_plane_state *state,
2292                                              int color_plane,
2293                                              u32 old_offset, u32 new_offset)
2294 {
2295         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2296                                            state->base.rotation,
2297                                            state->color_plane[color_plane].stride,
2298                                            old_offset, new_offset);
2299 }
2300
2301 /*
2302  * Computes the aligned offset to the base tile and adjusts
2303  * x, y. bytes per pixel is assumed to be a power-of-two.
2304  *
2305  * In the 90/270 rotated case, x and y are assumed
2306  * to be already rotated to match the rotated GTT view, and
2307  * pitch is the tile_height aligned framebuffer height.
2308  *
2309  * This function is used when computing the derived information
2310  * under intel_framebuffer, so using any of that information
2311  * here is not allowed. Anything under drm_framebuffer can be
2312  * used. This is why the user has to pass in the pitch since it
2313  * is specified in the rotated orientation.
2314  */
2315 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2316                                         int *x, int *y,
2317                                         const struct drm_framebuffer *fb,
2318                                         int color_plane,
2319                                         unsigned int pitch,
2320                                         unsigned int rotation,
2321                                         u32 alignment)
2322 {
2323         unsigned int cpp = fb->format->cpp[color_plane];
2324         u32 offset, offset_aligned;
2325
2326         if (alignment)
2327                 alignment--;
2328
2329         if (!is_surface_linear(fb->modifier, color_plane)) {
2330                 unsigned int tile_size, tile_width, tile_height;
2331                 unsigned int tile_rows, tiles, pitch_tiles;
2332
2333                 tile_size = intel_tile_size(dev_priv);
2334                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2335
2336                 if (drm_rotation_90_or_270(rotation)) {
2337                         pitch_tiles = pitch / tile_height;
2338                         swap(tile_width, tile_height);
2339                 } else {
2340                         pitch_tiles = pitch / (tile_width * cpp);
2341                 }
2342
2343                 tile_rows = *y / tile_height;
2344                 *y %= tile_height;
2345
2346                 tiles = *x / tile_width;
2347                 *x %= tile_width;
2348
2349                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2350                 offset_aligned = offset & ~alignment;
2351
2352                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2353                                          tile_size, pitch_tiles,
2354                                          offset, offset_aligned);
2355         } else {
2356                 offset = *y * pitch + *x * cpp;
2357                 offset_aligned = offset & ~alignment;
2358
2359                 *y = (offset & alignment) / pitch;
2360                 *x = ((offset & alignment) - *y * pitch) / cpp;
2361         }
2362
2363         return offset_aligned;
2364 }
2365
2366 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2367                                               const struct intel_plane_state *state,
2368                                               int color_plane)
2369 {
2370         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2371         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2372         const struct drm_framebuffer *fb = state->base.fb;
2373         unsigned int rotation = state->base.rotation;
2374         int pitch = state->color_plane[color_plane].stride;
2375         u32 alignment;
2376
2377         if (intel_plane->id == PLANE_CURSOR)
2378                 alignment = intel_cursor_alignment(dev_priv);
2379         else
2380                 alignment = intel_surf_alignment(fb, color_plane);
2381
2382         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2383                                             pitch, rotation, alignment);
2384 }
2385
2386 /* Convert the fb->offset[] into x/y offsets */
2387 static int intel_fb_offset_to_xy(int *x, int *y,
2388                                  const struct drm_framebuffer *fb,
2389                                  int color_plane)
2390 {
2391         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2392         unsigned int height;
2393
2394         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2395             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2396                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2397                               fb->offsets[color_plane], color_plane);
2398                 return -EINVAL;
2399         }
2400
2401         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2402         height = ALIGN(height, intel_tile_height(fb, color_plane));
2403
2404         /* Catch potential overflows early */
2405         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2406                             fb->offsets[color_plane])) {
2407                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2408                               fb->offsets[color_plane], fb->pitches[color_plane],
2409                               color_plane);
2410                 return -ERANGE;
2411         }
2412
2413         *x = 0;
2414         *y = 0;
2415
2416         intel_adjust_aligned_offset(x, y,
2417                                     fb, color_plane, DRM_MODE_ROTATE_0,
2418                                     fb->pitches[color_plane],
2419                                     fb->offsets[color_plane], 0);
2420
2421         return 0;
2422 }
2423
2424 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2425 {
2426         switch (fb_modifier) {
2427         case I915_FORMAT_MOD_X_TILED:
2428                 return I915_TILING_X;
2429         case I915_FORMAT_MOD_Y_TILED:
2430         case I915_FORMAT_MOD_Y_TILED_CCS:
2431                 return I915_TILING_Y;
2432         default:
2433                 return I915_TILING_NONE;
2434         }
2435 }
2436
2437 /*
2438  * From the Sky Lake PRM:
2439  * "The Color Control Surface (CCS) contains the compression status of
2440  *  the cache-line pairs. The compression state of the cache-line pair
2441  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2442  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2443  *  cache-line-pairs. CCS is always Y tiled."
2444  *
2445  * Since cache line pairs refers to horizontally adjacent cache lines,
2446  * each cache line in the CCS corresponds to an area of 32x16 cache
2447  * lines on the main surface. Since each pixel is 4 bytes, this gives
2448  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2449  * main surface.
2450  */
2451 static const struct drm_format_info ccs_formats[] = {
2452         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2453           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2454         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2455           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2456         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2457           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2458         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2459           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2460 };
2461
2462 static const struct drm_format_info *
2463 lookup_format_info(const struct drm_format_info formats[],
2464                    int num_formats, u32 format)
2465 {
2466         int i;
2467
2468         for (i = 0; i < num_formats; i++) {
2469                 if (formats[i].format == format)
2470                         return &formats[i];
2471         }
2472
2473         return NULL;
2474 }
2475
2476 static const struct drm_format_info *
2477 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2478 {
2479         switch (cmd->modifier[0]) {
2480         case I915_FORMAT_MOD_Y_TILED_CCS:
2481         case I915_FORMAT_MOD_Yf_TILED_CCS:
2482                 return lookup_format_info(ccs_formats,
2483                                           ARRAY_SIZE(ccs_formats),
2484                                           cmd->pixel_format);
2485         default:
2486                 return NULL;
2487         }
2488 }
2489
2490 bool is_ccs_modifier(u64 modifier)
2491 {
2492         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2493                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2494 }
2495
2496 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2497                               u32 pixel_format, u64 modifier)
2498 {
2499         struct intel_crtc *crtc;
2500         struct intel_plane *plane;
2501
2502         /*
2503          * We assume the primary plane for pipe A has
2504          * the highest stride limits of them all.
2505          */
2506         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2507         plane = to_intel_plane(crtc->base.primary);
2508
2509         return plane->max_stride(plane, pixel_format, modifier,
2510                                  DRM_MODE_ROTATE_0);
2511 }
2512
2513 static
2514 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2515                         u32 pixel_format, u64 modifier)
2516 {
2517         /*
2518          * Arbitrary limit for gen4+ chosen to match the
2519          * render engine max stride.
2520          *
2521          * The new CCS hash mode makes remapping impossible
2522          */
2523         if (!is_ccs_modifier(modifier)) {
2524                 if (INTEL_GEN(dev_priv) >= 7)
2525                         return 256*1024;
2526                 else if (INTEL_GEN(dev_priv) >= 4)
2527                         return 128*1024;
2528         }
2529
2530         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2531 }
2532
2533 static u32
2534 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2535 {
2536         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2537
2538         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2539                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2540                                                            fb->format->format,
2541                                                            fb->modifier);
2542
2543                 /*
2544                  * To make remapping with linear generally feasible
2545                  * we need the stride to be page aligned.
2546                  */
2547                 if (fb->pitches[color_plane] > max_stride)
2548                         return intel_tile_size(dev_priv);
2549                 else
2550                         return 64;
2551         } else {
2552                 return intel_tile_width_bytes(fb, color_plane);
2553         }
2554 }
2555
2556 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2557 {
2558         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2559         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2560         const struct drm_framebuffer *fb = plane_state->base.fb;
2561         int i;
2562
2563         /* We don't want to deal with remapping with cursors */
2564         if (plane->id == PLANE_CURSOR)
2565                 return false;
2566
2567         /*
2568          * The display engine limits already match/exceed the
2569          * render engine limits, so not much point in remapping.
2570          * Would also need to deal with the fence POT alignment
2571          * and gen2 2KiB GTT tile size.
2572          */
2573         if (INTEL_GEN(dev_priv) < 4)
2574                 return false;
2575
2576         /*
2577          * The new CCS hash mode isn't compatible with remapping as
2578          * the virtual address of the pages affects the compressed data.
2579          */
2580         if (is_ccs_modifier(fb->modifier))
2581                 return false;
2582
2583         /* Linear needs a page aligned stride for remapping */
2584         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2585                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2586
2587                 for (i = 0; i < fb->format->num_planes; i++) {
2588                         if (fb->pitches[i] & alignment)
2589                                 return false;
2590                 }
2591         }
2592
2593         return true;
2594 }
2595
2596 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2597 {
2598         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2599         const struct drm_framebuffer *fb = plane_state->base.fb;
2600         unsigned int rotation = plane_state->base.rotation;
2601         u32 stride, max_stride;
2602
2603         /*
2604          * No remapping for invisible planes since we don't have
2605          * an actual source viewport to remap.
2606          */
2607         if (!plane_state->base.visible)
2608                 return false;
2609
2610         if (!intel_plane_can_remap(plane_state))
2611                 return false;
2612
2613         /*
2614          * FIXME: aux plane limits on gen9+ are
2615          * unclear in Bspec, for now no checking.
2616          */
2617         stride = intel_fb_pitch(fb, 0, rotation);
2618         max_stride = plane->max_stride(plane, fb->format->format,
2619                                        fb->modifier, rotation);
2620
2621         return stride > max_stride;
2622 }
2623
2624 static int
2625 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2626                    struct drm_framebuffer *fb)
2627 {
2628         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2629         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2630         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2631         u32 gtt_offset_rotated = 0;
2632         unsigned int max_size = 0;
2633         int i, num_planes = fb->format->num_planes;
2634         unsigned int tile_size = intel_tile_size(dev_priv);
2635
2636         for (i = 0; i < num_planes; i++) {
2637                 unsigned int width, height;
2638                 unsigned int cpp, size;
2639                 u32 offset;
2640                 int x, y;
2641                 int ret;
2642
2643                 cpp = fb->format->cpp[i];
2644                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2645                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2646
2647                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2648                 if (ret) {
2649                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2650                                       i, fb->offsets[i]);
2651                         return ret;
2652                 }
2653
2654                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2655                         int hsub = fb->format->hsub;
2656                         int vsub = fb->format->vsub;
2657                         int tile_width, tile_height;
2658                         int main_x, main_y;
2659                         int ccs_x, ccs_y;
2660
2661                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2662                         tile_width *= hsub;
2663                         tile_height *= vsub;
2664
2665                         ccs_x = (x * hsub) % tile_width;
2666                         ccs_y = (y * vsub) % tile_height;
2667                         main_x = intel_fb->normal[0].x % tile_width;
2668                         main_y = intel_fb->normal[0].y % tile_height;
2669
2670                         /*
2671                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2672                          * x/y offsets must match between CCS and the main surface.
2673                          */
2674                         if (main_x != ccs_x || main_y != ccs_y) {
2675                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2676                                               main_x, main_y,
2677                                               ccs_x, ccs_y,
2678                                               intel_fb->normal[0].x,
2679                                               intel_fb->normal[0].y,
2680                                               x, y);
2681                                 return -EINVAL;
2682                         }
2683                 }
2684
2685                 /*
2686                  * The fence (if used) is aligned to the start of the object
2687                  * so having the framebuffer wrap around across the edge of the
2688                  * fenced region doesn't really work. We have no API to configure
2689                  * the fence start offset within the object (nor could we probably
2690                  * on gen2/3). So it's just easier if we just require that the
2691                  * fb layout agrees with the fence layout. We already check that the
2692                  * fb stride matches the fence stride elsewhere.
2693                  */
2694                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2695                     (x + width) * cpp > fb->pitches[i]) {
2696                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2697                                       i, fb->offsets[i]);
2698                         return -EINVAL;
2699                 }
2700
2701                 /*
2702                  * First pixel of the framebuffer from
2703                  * the start of the normal gtt mapping.
2704                  */
2705                 intel_fb->normal[i].x = x;
2706                 intel_fb->normal[i].y = y;
2707
2708                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2709                                                       fb->pitches[i],
2710                                                       DRM_MODE_ROTATE_0,
2711                                                       tile_size);
2712                 offset /= tile_size;
2713
2714                 if (!is_surface_linear(fb->modifier, i)) {
2715                         unsigned int tile_width, tile_height;
2716                         unsigned int pitch_tiles;
2717                         struct drm_rect r;
2718
2719                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2720
2721                         rot_info->plane[i].offset = offset;
2722                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2723                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2724                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2725
2726                         intel_fb->rotated[i].pitch =
2727                                 rot_info->plane[i].height * tile_height;
2728
2729                         /* how many tiles does this plane need */
2730                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2731                         /*
2732                          * If the plane isn't horizontally tile aligned,
2733                          * we need one more tile.
2734                          */
2735                         if (x != 0)
2736                                 size++;
2737
2738                         /* rotate the x/y offsets to match the GTT view */
2739                         r.x1 = x;
2740                         r.y1 = y;
2741                         r.x2 = x + width;
2742                         r.y2 = y + height;
2743                         drm_rect_rotate(&r,
2744                                         rot_info->plane[i].width * tile_width,
2745                                         rot_info->plane[i].height * tile_height,
2746                                         DRM_MODE_ROTATE_270);
2747                         x = r.x1;
2748                         y = r.y1;
2749
2750                         /* rotate the tile dimensions to match the GTT view */
2751                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2752                         swap(tile_width, tile_height);
2753
2754                         /*
2755                          * We only keep the x/y offsets, so push all of the
2756                          * gtt offset into the x/y offsets.
2757                          */
2758                         intel_adjust_tile_offset(&x, &y,
2759                                                  tile_width, tile_height,
2760                                                  tile_size, pitch_tiles,
2761                                                  gtt_offset_rotated * tile_size, 0);
2762
2763                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2764
2765                         /*
2766                          * First pixel of the framebuffer from
2767                          * the start of the rotated gtt mapping.
2768                          */
2769                         intel_fb->rotated[i].x = x;
2770                         intel_fb->rotated[i].y = y;
2771                 } else {
2772                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2773                                             x * cpp, tile_size);
2774                 }
2775
2776                 /* how many tiles in total needed in the bo */
2777                 max_size = max(max_size, offset + size);
2778         }
2779
2780         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2781                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2782                               mul_u32_u32(max_size, tile_size), obj->base.size);
2783                 return -EINVAL;
2784         }
2785
2786         return 0;
2787 }
2788
2789 static void
2790 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2791 {
2792         struct drm_i915_private *dev_priv =
2793                 to_i915(plane_state->base.plane->dev);
2794         struct drm_framebuffer *fb = plane_state->base.fb;
2795         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2796         struct intel_rotation_info *info = &plane_state->view.rotated;
2797         unsigned int rotation = plane_state->base.rotation;
2798         int i, num_planes = fb->format->num_planes;
2799         unsigned int tile_size = intel_tile_size(dev_priv);
2800         unsigned int src_x, src_y;
2801         unsigned int src_w, src_h;
2802         u32 gtt_offset = 0;
2803
2804         memset(&plane_state->view, 0, sizeof(plane_state->view));
2805         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2806                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2807
2808         src_x = plane_state->base.src.x1 >> 16;
2809         src_y = plane_state->base.src.y1 >> 16;
2810         src_w = drm_rect_width(&plane_state->base.src) >> 16;
2811         src_h = drm_rect_height(&plane_state->base.src) >> 16;
2812
2813         WARN_ON(is_ccs_modifier(fb->modifier));
2814
2815         /* Make src coordinates relative to the viewport */
2816         drm_rect_translate(&plane_state->base.src,
2817                            -(src_x << 16), -(src_y << 16));
2818
2819         /* Rotate src coordinates to match rotated GTT view */
2820         if (drm_rotation_90_or_270(rotation))
2821                 drm_rect_rotate(&plane_state->base.src,
2822                                 src_w << 16, src_h << 16,
2823                                 DRM_MODE_ROTATE_270);
2824
2825         for (i = 0; i < num_planes; i++) {
2826                 unsigned int hsub = i ? fb->format->hsub : 1;
2827                 unsigned int vsub = i ? fb->format->vsub : 1;
2828                 unsigned int cpp = fb->format->cpp[i];
2829                 unsigned int tile_width, tile_height;
2830                 unsigned int width, height;
2831                 unsigned int pitch_tiles;
2832                 unsigned int x, y;
2833                 u32 offset;
2834
2835                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2836
2837                 x = src_x / hsub;
2838                 y = src_y / vsub;
2839                 width = src_w / hsub;
2840                 height = src_h / vsub;
2841
2842                 /*
2843                  * First pixel of the src viewport from the
2844                  * start of the normal gtt mapping.
2845                  */
2846                 x += intel_fb->normal[i].x;
2847                 y += intel_fb->normal[i].y;
2848
2849                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2850                                                       fb, i, fb->pitches[i],
2851                                                       DRM_MODE_ROTATE_0, tile_size);
2852                 offset /= tile_size;
2853
2854                 info->plane[i].offset = offset;
2855                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2856                                                      tile_width * cpp);
2857                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2858                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2859
2860                 if (drm_rotation_90_or_270(rotation)) {
2861                         struct drm_rect r;
2862
2863                         /* rotate the x/y offsets to match the GTT view */
2864                         r.x1 = x;
2865                         r.y1 = y;
2866                         r.x2 = x + width;
2867                         r.y2 = y + height;
2868                         drm_rect_rotate(&r,
2869                                         info->plane[i].width * tile_width,
2870                                         info->plane[i].height * tile_height,
2871                                         DRM_MODE_ROTATE_270);
2872                         x = r.x1;
2873                         y = r.y1;
2874
2875                         pitch_tiles = info->plane[i].height;
2876                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2877
2878                         /* rotate the tile dimensions to match the GTT view */
2879                         swap(tile_width, tile_height);
2880                 } else {
2881                         pitch_tiles = info->plane[i].width;
2882                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2883                 }
2884
2885                 /*
2886                  * We only keep the x/y offsets, so push all of the
2887                  * gtt offset into the x/y offsets.
2888                  */
2889                 intel_adjust_tile_offset(&x, &y,
2890                                          tile_width, tile_height,
2891                                          tile_size, pitch_tiles,
2892                                          gtt_offset * tile_size, 0);
2893
2894                 gtt_offset += info->plane[i].width * info->plane[i].height;
2895
2896                 plane_state->color_plane[i].offset = 0;
2897                 plane_state->color_plane[i].x = x;
2898                 plane_state->color_plane[i].y = y;
2899         }
2900 }
2901
2902 static int
2903 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2904 {
2905         const struct intel_framebuffer *fb =
2906                 to_intel_framebuffer(plane_state->base.fb);
2907         unsigned int rotation = plane_state->base.rotation;
2908         int i, num_planes;
2909
2910         if (!fb)
2911                 return 0;
2912
2913         num_planes = fb->base.format->num_planes;
2914
2915         if (intel_plane_needs_remap(plane_state)) {
2916                 intel_plane_remap_gtt(plane_state);
2917
2918                 /*
2919                  * Sometimes even remapping can't overcome
2920                  * the stride limitations :( Can happen with
2921                  * big plane sizes and suitably misaligned
2922                  * offsets.
2923                  */
2924                 return intel_plane_check_stride(plane_state);
2925         }
2926
2927         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2928
2929         for (i = 0; i < num_planes; i++) {
2930                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2931                 plane_state->color_plane[i].offset = 0;
2932
2933                 if (drm_rotation_90_or_270(rotation)) {
2934                         plane_state->color_plane[i].x = fb->rotated[i].x;
2935                         plane_state->color_plane[i].y = fb->rotated[i].y;
2936                 } else {
2937                         plane_state->color_plane[i].x = fb->normal[i].x;
2938                         plane_state->color_plane[i].y = fb->normal[i].y;
2939                 }
2940         }
2941
2942         /* Rotate src coordinates to match rotated GTT view */
2943         if (drm_rotation_90_or_270(rotation))
2944                 drm_rect_rotate(&plane_state->base.src,
2945                                 fb->base.width << 16, fb->base.height << 16,
2946                                 DRM_MODE_ROTATE_270);
2947
2948         return intel_plane_check_stride(plane_state);
2949 }
2950
2951 static int i9xx_format_to_fourcc(int format)
2952 {
2953         switch (format) {
2954         case DISPPLANE_8BPP:
2955                 return DRM_FORMAT_C8;
2956         case DISPPLANE_BGRX555:
2957                 return DRM_FORMAT_XRGB1555;
2958         case DISPPLANE_BGRX565:
2959                 return DRM_FORMAT_RGB565;
2960         default:
2961         case DISPPLANE_BGRX888:
2962                 return DRM_FORMAT_XRGB8888;
2963         case DISPPLANE_RGBX888:
2964                 return DRM_FORMAT_XBGR8888;
2965         case DISPPLANE_BGRX101010:
2966                 return DRM_FORMAT_XRGB2101010;
2967         case DISPPLANE_RGBX101010:
2968                 return DRM_FORMAT_XBGR2101010;
2969         }
2970 }
2971
2972 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2973 {
2974         switch (format) {
2975         case PLANE_CTL_FORMAT_RGB_565:
2976                 return DRM_FORMAT_RGB565;
2977         case PLANE_CTL_FORMAT_NV12:
2978                 return DRM_FORMAT_NV12;
2979         case PLANE_CTL_FORMAT_P010:
2980                 return DRM_FORMAT_P010;
2981         case PLANE_CTL_FORMAT_P012:
2982                 return DRM_FORMAT_P012;
2983         case PLANE_CTL_FORMAT_P016:
2984                 return DRM_FORMAT_P016;
2985         case PLANE_CTL_FORMAT_Y210:
2986                 return DRM_FORMAT_Y210;
2987         case PLANE_CTL_FORMAT_Y212:
2988                 return DRM_FORMAT_Y212;
2989         case PLANE_CTL_FORMAT_Y216:
2990                 return DRM_FORMAT_Y216;
2991         case PLANE_CTL_FORMAT_Y410:
2992                 return DRM_FORMAT_XVYU2101010;
2993         case PLANE_CTL_FORMAT_Y412:
2994                 return DRM_FORMAT_XVYU12_16161616;
2995         case PLANE_CTL_FORMAT_Y416:
2996                 return DRM_FORMAT_XVYU16161616;
2997         default:
2998         case PLANE_CTL_FORMAT_XRGB_8888:
2999                 if (rgb_order) {
3000                         if (alpha)
3001                                 return DRM_FORMAT_ABGR8888;
3002                         else
3003                                 return DRM_FORMAT_XBGR8888;
3004                 } else {
3005                         if (alpha)
3006                                 return DRM_FORMAT_ARGB8888;
3007                         else
3008                                 return DRM_FORMAT_XRGB8888;
3009                 }
3010         case PLANE_CTL_FORMAT_XRGB_2101010:
3011                 if (rgb_order)
3012                         return DRM_FORMAT_XBGR2101010;
3013                 else
3014                         return DRM_FORMAT_XRGB2101010;
3015         case PLANE_CTL_FORMAT_XRGB_16161616F:
3016                 if (rgb_order) {
3017                         if (alpha)
3018                                 return DRM_FORMAT_ABGR16161616F;
3019                         else
3020                                 return DRM_FORMAT_XBGR16161616F;
3021                 } else {
3022                         if (alpha)
3023                                 return DRM_FORMAT_ARGB16161616F;
3024                         else
3025                                 return DRM_FORMAT_XRGB16161616F;
3026                 }
3027         }
3028 }
3029
3030 static bool
3031 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3032                               struct intel_initial_plane_config *plane_config)
3033 {
3034         struct drm_device *dev = crtc->base.dev;
3035         struct drm_i915_private *dev_priv = to_i915(dev);
3036         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3037         struct drm_framebuffer *fb = &plane_config->fb->base;
3038         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3039         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3040                                     PAGE_SIZE);
3041         struct drm_i915_gem_object *obj;
3042         bool ret = false;
3043
3044         size_aligned -= base_aligned;
3045
3046         if (plane_config->size == 0)
3047                 return false;
3048
3049         /* If the FB is too big, just don't use it since fbdev is not very
3050          * important and we should probably use that space with FBC or other
3051          * features. */
3052         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3053                 return false;
3054
3055         switch (fb->modifier) {
3056         case DRM_FORMAT_MOD_LINEAR:
3057         case I915_FORMAT_MOD_X_TILED:
3058         case I915_FORMAT_MOD_Y_TILED:
3059                 break;
3060         default:
3061                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3062                                  fb->modifier);
3063                 return false;
3064         }
3065
3066         mutex_lock(&dev->struct_mutex);
3067         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3068                                                              base_aligned,
3069                                                              base_aligned,
3070                                                              size_aligned);
3071         mutex_unlock(&dev->struct_mutex);
3072         if (!obj)
3073                 return false;
3074
3075         switch (plane_config->tiling) {
3076         case I915_TILING_NONE:
3077                 break;
3078         case I915_TILING_X:
3079         case I915_TILING_Y:
3080                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3081                 break;
3082         default:
3083                 MISSING_CASE(plane_config->tiling);
3084                 goto out;
3085         }
3086
3087         mode_cmd.pixel_format = fb->format->format;
3088         mode_cmd.width = fb->width;
3089         mode_cmd.height = fb->height;
3090         mode_cmd.pitches[0] = fb->pitches[0];
3091         mode_cmd.modifier[0] = fb->modifier;
3092         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3093
3094         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3095                 DRM_DEBUG_KMS("intel fb init failed\n");
3096                 goto out;
3097         }
3098
3099
3100         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3101         ret = true;
3102 out:
3103         i915_gem_object_put(obj);
3104         return ret;
3105 }
3106
3107 static void
3108 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3109                         struct intel_plane_state *plane_state,
3110                         bool visible)
3111 {
3112         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3113
3114         plane_state->base.visible = visible;
3115
3116         if (visible)
3117                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
3118         else
3119                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
3120 }
3121
3122 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3123 {
3124         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3125         struct drm_plane *plane;
3126
3127         /*
3128          * Active_planes aliases if multiple "primary" or cursor planes
3129          * have been used on the same (or wrong) pipe. plane_mask uses
3130          * unique ids, hence we can use that to reconstruct active_planes.
3131          */
3132         crtc_state->active_planes = 0;
3133
3134         drm_for_each_plane_mask(plane, &dev_priv->drm,
3135                                 crtc_state->base.plane_mask)
3136                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3137 }
3138
3139 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3140                                          struct intel_plane *plane)
3141 {
3142         struct intel_crtc_state *crtc_state =
3143                 to_intel_crtc_state(crtc->base.state);
3144         struct intel_plane_state *plane_state =
3145                 to_intel_plane_state(plane->base.state);
3146
3147         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3148                       plane->base.base.id, plane->base.name,
3149                       crtc->base.base.id, crtc->base.name);
3150
3151         intel_set_plane_visible(crtc_state, plane_state, false);
3152         fixup_active_planes(crtc_state);
3153         crtc_state->data_rate[plane->id] = 0;
3154
3155         if (plane->id == PLANE_PRIMARY)
3156                 intel_pre_disable_primary_noatomic(&crtc->base);
3157
3158         intel_disable_plane(plane, crtc_state);
3159 }
3160
3161 static struct intel_frontbuffer *
3162 to_intel_frontbuffer(struct drm_framebuffer *fb)
3163 {
3164         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3165 }
3166
3167 static void
3168 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3169                              struct intel_initial_plane_config *plane_config)
3170 {
3171         struct drm_device *dev = intel_crtc->base.dev;
3172         struct drm_i915_private *dev_priv = to_i915(dev);
3173         struct drm_crtc *c;
3174         struct drm_plane *primary = intel_crtc->base.primary;
3175         struct drm_plane_state *plane_state = primary->state;
3176         struct intel_plane *intel_plane = to_intel_plane(primary);
3177         struct intel_plane_state *intel_state =
3178                 to_intel_plane_state(plane_state);
3179         struct drm_framebuffer *fb;
3180
3181         if (!plane_config->fb)
3182                 return;
3183
3184         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3185                 fb = &plane_config->fb->base;
3186                 goto valid_fb;
3187         }
3188
3189         kfree(plane_config->fb);
3190
3191         /*
3192          * Failed to alloc the obj, check to see if we should share
3193          * an fb with another CRTC instead
3194          */
3195         for_each_crtc(dev, c) {
3196                 struct intel_plane_state *state;
3197
3198                 if (c == &intel_crtc->base)
3199                         continue;
3200
3201                 if (!to_intel_crtc(c)->active)
3202                         continue;
3203
3204                 state = to_intel_plane_state(c->primary->state);
3205                 if (!state->vma)
3206                         continue;
3207
3208                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3209                         fb = state->base.fb;
3210                         drm_framebuffer_get(fb);
3211                         goto valid_fb;
3212                 }
3213         }
3214
3215         /*
3216          * We've failed to reconstruct the BIOS FB.  Current display state
3217          * indicates that the primary plane is visible, but has a NULL FB,
3218          * which will lead to problems later if we don't fix it up.  The
3219          * simplest solution is to just disable the primary plane now and
3220          * pretend the BIOS never had it enabled.
3221          */
3222         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3223
3224         return;
3225
3226 valid_fb:
3227         intel_state->base.rotation = plane_config->rotation;
3228         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3229                                 intel_state->base.rotation);
3230         intel_state->color_plane[0].stride =
3231                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
3232
3233         mutex_lock(&dev->struct_mutex);
3234         intel_state->vma =
3235                 intel_pin_and_fence_fb_obj(fb,
3236                                            &intel_state->view,
3237                                            intel_plane_uses_fence(intel_state),
3238                                            &intel_state->flags);
3239         mutex_unlock(&dev->struct_mutex);
3240         if (IS_ERR(intel_state->vma)) {
3241                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3242                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3243
3244                 intel_state->vma = NULL;
3245                 drm_framebuffer_put(fb);
3246                 return;
3247         }
3248
3249         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3250
3251         plane_state->src_x = 0;
3252         plane_state->src_y = 0;
3253         plane_state->src_w = fb->width << 16;
3254         plane_state->src_h = fb->height << 16;
3255
3256         plane_state->crtc_x = 0;
3257         plane_state->crtc_y = 0;
3258         plane_state->crtc_w = fb->width;
3259         plane_state->crtc_h = fb->height;
3260
3261         intel_state->base.src = drm_plane_state_src(plane_state);
3262         intel_state->base.dst = drm_plane_state_dest(plane_state);
3263
3264         if (plane_config->tiling)
3265                 dev_priv->preserve_bios_swizzle = true;
3266
3267         plane_state->fb = fb;
3268         plane_state->crtc = &intel_crtc->base;
3269
3270         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3271                   &to_intel_frontbuffer(fb)->bits);
3272 }
3273
3274 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3275                                int color_plane,
3276                                unsigned int rotation)
3277 {
3278         int cpp = fb->format->cpp[color_plane];
3279
3280         switch (fb->modifier) {
3281         case DRM_FORMAT_MOD_LINEAR:
3282         case I915_FORMAT_MOD_X_TILED:
3283                 return 4096;
3284         case I915_FORMAT_MOD_Y_TILED_CCS:
3285         case I915_FORMAT_MOD_Yf_TILED_CCS:
3286                 /* FIXME AUX plane? */
3287         case I915_FORMAT_MOD_Y_TILED:
3288         case I915_FORMAT_MOD_Yf_TILED:
3289                 if (cpp == 8)
3290                         return 2048;
3291                 else
3292                         return 4096;
3293         default:
3294                 MISSING_CASE(fb->modifier);
3295                 return 2048;
3296         }
3297 }
3298
3299 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3300                                int color_plane,
3301                                unsigned int rotation)
3302 {
3303         int cpp = fb->format->cpp[color_plane];
3304
3305         switch (fb->modifier) {
3306         case DRM_FORMAT_MOD_LINEAR:
3307         case I915_FORMAT_MOD_X_TILED:
3308                 if (cpp == 8)
3309                         return 4096;
3310                 else
3311                         return 5120;
3312         case I915_FORMAT_MOD_Y_TILED_CCS:
3313         case I915_FORMAT_MOD_Yf_TILED_CCS:
3314                 /* FIXME AUX plane? */
3315         case I915_FORMAT_MOD_Y_TILED:
3316         case I915_FORMAT_MOD_Yf_TILED:
3317                 if (cpp == 8)
3318                         return 2048;
3319                 else
3320                         return 5120;
3321         default:
3322                 MISSING_CASE(fb->modifier);
3323                 return 2048;
3324         }
3325 }
3326
3327 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3328                                int color_plane,
3329                                unsigned int rotation)
3330 {
3331         return 5120;
3332 }
3333
3334 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3335                                            int main_x, int main_y, u32 main_offset)
3336 {
3337         const struct drm_framebuffer *fb = plane_state->base.fb;
3338         int hsub = fb->format->hsub;
3339         int vsub = fb->format->vsub;
3340         int aux_x = plane_state->color_plane[1].x;
3341         int aux_y = plane_state->color_plane[1].y;
3342         u32 aux_offset = plane_state->color_plane[1].offset;
3343         u32 alignment = intel_surf_alignment(fb, 1);
3344
3345         while (aux_offset >= main_offset && aux_y <= main_y) {
3346                 int x, y;
3347
3348                 if (aux_x == main_x && aux_y == main_y)
3349                         break;
3350
3351                 if (aux_offset == 0)
3352                         break;
3353
3354                 x = aux_x / hsub;
3355                 y = aux_y / vsub;
3356                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
3357                                                                aux_offset, aux_offset - alignment);
3358                 aux_x = x * hsub + aux_x % hsub;
3359                 aux_y = y * vsub + aux_y % vsub;
3360         }
3361
3362         if (aux_x != main_x || aux_y != main_y)
3363                 return false;
3364
3365         plane_state->color_plane[1].offset = aux_offset;
3366         plane_state->color_plane[1].x = aux_x;
3367         plane_state->color_plane[1].y = aux_y;
3368
3369         return true;
3370 }
3371
3372 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3373 {
3374         struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev);
3375         const struct drm_framebuffer *fb = plane_state->base.fb;
3376         unsigned int rotation = plane_state->base.rotation;
3377         int x = plane_state->base.src.x1 >> 16;
3378         int y = plane_state->base.src.y1 >> 16;
3379         int w = drm_rect_width(&plane_state->base.src) >> 16;
3380         int h = drm_rect_height(&plane_state->base.src) >> 16;
3381         int max_width;
3382         int max_height = 4096;
3383         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3384
3385         if (INTEL_GEN(dev_priv) >= 11)
3386                 max_width = icl_max_plane_width(fb, 0, rotation);
3387         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3388                 max_width = glk_max_plane_width(fb, 0, rotation);
3389         else
3390                 max_width = skl_max_plane_width(fb, 0, rotation);
3391
3392         if (w > max_width || h > max_height) {
3393                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3394                               w, h, max_width, max_height);
3395                 return -EINVAL;
3396         }
3397
3398         intel_add_fb_offsets(&x, &y, plane_state, 0);
3399         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3400         alignment = intel_surf_alignment(fb, 0);
3401
3402         /*
3403          * AUX surface offset is specified as the distance from the
3404          * main surface offset, and it must be non-negative. Make
3405          * sure that is what we will get.
3406          */
3407         if (offset > aux_offset)
3408                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3409                                                            offset, aux_offset & ~(alignment - 1));
3410
3411         /*
3412          * When using an X-tiled surface, the plane blows up
3413          * if the x offset + width exceed the stride.
3414          *
3415          * TODO: linear and Y-tiled seem fine, Yf untested,
3416          */
3417         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3418                 int cpp = fb->format->cpp[0];
3419
3420                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3421                         if (offset == 0) {
3422                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3423                                 return -EINVAL;
3424                         }
3425
3426                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3427                                                                    offset, offset - alignment);
3428                 }
3429         }
3430
3431         /*
3432          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3433          * they match with the main surface x/y offsets.
3434          */
3435         if (is_ccs_modifier(fb->modifier)) {
3436                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3437                         if (offset == 0)
3438                                 break;
3439
3440                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3441                                                                    offset, offset - alignment);
3442                 }
3443
3444                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3445                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3446                         return -EINVAL;
3447                 }
3448         }
3449
3450         plane_state->color_plane[0].offset = offset;
3451         plane_state->color_plane[0].x = x;
3452         plane_state->color_plane[0].y = y;
3453
3454         /*
3455          * Put the final coordinates back so that the src
3456          * coordinate checks will see the right values.
3457          */
3458         drm_rect_translate(&plane_state->base.src,
3459                            (x << 16) - plane_state->base.src.x1,
3460                            (y << 16) - plane_state->base.src.y1);
3461
3462         return 0;
3463 }
3464
3465 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3466 {
3467         const struct drm_framebuffer *fb = plane_state->base.fb;
3468         unsigned int rotation = plane_state->base.rotation;
3469         int max_width = skl_max_plane_width(fb, 1, rotation);
3470         int max_height = 4096;
3471         int x = plane_state->base.src.x1 >> 17;
3472         int y = plane_state->base.src.y1 >> 17;
3473         int w = drm_rect_width(&plane_state->base.src) >> 17;
3474         int h = drm_rect_height(&plane_state->base.src) >> 17;
3475         u32 offset;
3476
3477         intel_add_fb_offsets(&x, &y, plane_state, 1);
3478         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3479
3480         /* FIXME not quite sure how/if these apply to the chroma plane */
3481         if (w > max_width || h > max_height) {
3482                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3483                               w, h, max_width, max_height);
3484                 return -EINVAL;
3485         }
3486
3487         plane_state->color_plane[1].offset = offset;
3488         plane_state->color_plane[1].x = x;
3489         plane_state->color_plane[1].y = y;
3490
3491         return 0;
3492 }
3493
3494 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3495 {
3496         const struct drm_framebuffer *fb = plane_state->base.fb;
3497         int src_x = plane_state->base.src.x1 >> 16;
3498         int src_y = plane_state->base.src.y1 >> 16;
3499         int hsub = fb->format->hsub;
3500         int vsub = fb->format->vsub;
3501         int x = src_x / hsub;
3502         int y = src_y / vsub;
3503         u32 offset;
3504
3505         intel_add_fb_offsets(&x, &y, plane_state, 1);
3506         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3507
3508         plane_state->color_plane[1].offset = offset;
3509         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3510         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3511
3512         return 0;
3513 }
3514
3515 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3516 {
3517         const struct drm_framebuffer *fb = plane_state->base.fb;
3518         int ret;
3519
3520         ret = intel_plane_compute_gtt(plane_state);
3521         if (ret)
3522                 return ret;
3523
3524         if (!plane_state->base.visible)
3525                 return 0;
3526
3527         /*
3528          * Handle the AUX surface first since
3529          * the main surface setup depends on it.
3530          */
3531         if (is_planar_yuv_format(fb->format->format)) {
3532                 ret = skl_check_nv12_aux_surface(plane_state);
3533                 if (ret)
3534                         return ret;
3535         } else if (is_ccs_modifier(fb->modifier)) {
3536                 ret = skl_check_ccs_aux_surface(plane_state);
3537                 if (ret)
3538                         return ret;
3539         } else {
3540                 plane_state->color_plane[1].offset = ~0xfff;
3541                 plane_state->color_plane[1].x = 0;
3542                 plane_state->color_plane[1].y = 0;
3543         }
3544
3545         ret = skl_check_main_surface(plane_state);
3546         if (ret)
3547                 return ret;
3548
3549         return 0;
3550 }
3551
3552 unsigned int
3553 i9xx_plane_max_stride(struct intel_plane *plane,
3554                       u32 pixel_format, u64 modifier,
3555                       unsigned int rotation)
3556 {
3557         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3558
3559         if (!HAS_GMCH(dev_priv)) {
3560                 return 32*1024;
3561         } else if (INTEL_GEN(dev_priv) >= 4) {
3562                 if (modifier == I915_FORMAT_MOD_X_TILED)
3563                         return 16*1024;
3564                 else
3565                         return 32*1024;
3566         } else if (INTEL_GEN(dev_priv) >= 3) {
3567                 if (modifier == I915_FORMAT_MOD_X_TILED)
3568                         return 8*1024;
3569                 else
3570                         return 16*1024;
3571         } else {
3572                 if (plane->i9xx_plane == PLANE_C)
3573                         return 4*1024;
3574                 else
3575                         return 8*1024;
3576         }
3577 }
3578
3579 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3580 {
3581         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3582         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3583         u32 dspcntr = 0;
3584
3585         if (crtc_state->gamma_enable)
3586                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3587
3588         if (crtc_state->csc_enable)
3589                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3590
3591         if (INTEL_GEN(dev_priv) < 5)
3592                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3593
3594         return dspcntr;
3595 }
3596
3597 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3598                           const struct intel_plane_state *plane_state)
3599 {
3600         struct drm_i915_private *dev_priv =
3601                 to_i915(plane_state->base.plane->dev);
3602         const struct drm_framebuffer *fb = plane_state->base.fb;
3603         unsigned int rotation = plane_state->base.rotation;
3604         u32 dspcntr;
3605
3606         dspcntr = DISPLAY_PLANE_ENABLE;
3607
3608         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3609             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
3610                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3611
3612         switch (fb->format->format) {
3613         case DRM_FORMAT_C8:
3614                 dspcntr |= DISPPLANE_8BPP;
3615                 break;
3616         case DRM_FORMAT_XRGB1555:
3617                 dspcntr |= DISPPLANE_BGRX555;
3618                 break;
3619         case DRM_FORMAT_RGB565:
3620                 dspcntr |= DISPPLANE_BGRX565;
3621                 break;
3622         case DRM_FORMAT_XRGB8888:
3623                 dspcntr |= DISPPLANE_BGRX888;
3624                 break;
3625         case DRM_FORMAT_XBGR8888:
3626                 dspcntr |= DISPPLANE_RGBX888;
3627                 break;
3628         case DRM_FORMAT_XRGB2101010:
3629                 dspcntr |= DISPPLANE_BGRX101010;
3630                 break;
3631         case DRM_FORMAT_XBGR2101010:
3632                 dspcntr |= DISPPLANE_RGBX101010;
3633                 break;
3634         default:
3635                 MISSING_CASE(fb->format->format);
3636                 return 0;
3637         }
3638
3639         if (INTEL_GEN(dev_priv) >= 4 &&
3640             fb->modifier == I915_FORMAT_MOD_X_TILED)
3641                 dspcntr |= DISPPLANE_TILED;
3642
3643         if (rotation & DRM_MODE_ROTATE_180)
3644                 dspcntr |= DISPPLANE_ROTATE_180;
3645
3646         if (rotation & DRM_MODE_REFLECT_X)
3647                 dspcntr |= DISPPLANE_MIRROR;
3648
3649         return dspcntr;
3650 }
3651
3652 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3653 {
3654         struct drm_i915_private *dev_priv =
3655                 to_i915(plane_state->base.plane->dev);
3656         int src_x, src_y;
3657         u32 offset;
3658         int ret;
3659
3660         ret = intel_plane_compute_gtt(plane_state);
3661         if (ret)
3662                 return ret;
3663
3664         if (!plane_state->base.visible)
3665                 return 0;
3666
3667         src_x = plane_state->base.src.x1 >> 16;
3668         src_y = plane_state->base.src.y1 >> 16;
3669
3670         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3671
3672         if (INTEL_GEN(dev_priv) >= 4)
3673                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3674                                                             plane_state, 0);
3675         else
3676                 offset = 0;
3677
3678         /*
3679          * Put the final coordinates back so that the src
3680          * coordinate checks will see the right values.
3681          */
3682         drm_rect_translate(&plane_state->base.src,
3683                            (src_x << 16) - plane_state->base.src.x1,
3684                            (src_y << 16) - plane_state->base.src.y1);
3685
3686         /* HSW/BDW do this automagically in hardware */
3687         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3688                 unsigned int rotation = plane_state->base.rotation;
3689                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3690                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3691
3692                 if (rotation & DRM_MODE_ROTATE_180) {
3693                         src_x += src_w - 1;
3694                         src_y += src_h - 1;
3695                 } else if (rotation & DRM_MODE_REFLECT_X) {
3696                         src_x += src_w - 1;
3697                 }
3698         }
3699
3700         plane_state->color_plane[0].offset = offset;
3701         plane_state->color_plane[0].x = src_x;
3702         plane_state->color_plane[0].y = src_y;
3703
3704         return 0;
3705 }
3706
3707 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
3708 {
3709         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3710         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3711
3712         if (IS_CHERRYVIEW(dev_priv))
3713                 return i9xx_plane == PLANE_B;
3714         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
3715                 return false;
3716         else if (IS_GEN(dev_priv, 4))
3717                 return i9xx_plane == PLANE_C;
3718         else
3719                 return i9xx_plane == PLANE_B ||
3720                         i9xx_plane == PLANE_C;
3721 }
3722
3723 static int
3724 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3725                  struct intel_plane_state *plane_state)
3726 {
3727         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3728         int ret;
3729
3730         ret = chv_plane_check_rotation(plane_state);
3731         if (ret)
3732                 return ret;
3733
3734         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3735                                                   &crtc_state->base,
3736                                                   DRM_PLANE_HELPER_NO_SCALING,
3737                                                   DRM_PLANE_HELPER_NO_SCALING,
3738                                                   i9xx_plane_has_windowing(plane),
3739                                                   true);
3740         if (ret)
3741                 return ret;
3742
3743         ret = i9xx_check_plane_surface(plane_state);
3744         if (ret)
3745                 return ret;
3746
3747         if (!plane_state->base.visible)
3748                 return 0;
3749
3750         ret = intel_plane_check_src_coordinates(plane_state);
3751         if (ret)
3752                 return ret;
3753
3754         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3755
3756         return 0;
3757 }
3758
3759 static void i9xx_update_plane(struct intel_plane *plane,
3760                               const struct intel_crtc_state *crtc_state,
3761                               const struct intel_plane_state *plane_state)
3762 {
3763         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3764         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3765         u32 linear_offset;
3766         int x = plane_state->color_plane[0].x;
3767         int y = plane_state->color_plane[0].y;
3768         int crtc_x = plane_state->base.dst.x1;
3769         int crtc_y = plane_state->base.dst.y1;
3770         int crtc_w = drm_rect_width(&plane_state->base.dst);
3771         int crtc_h = drm_rect_height(&plane_state->base.dst);
3772         unsigned long irqflags;
3773         u32 dspaddr_offset;
3774         u32 dspcntr;
3775
3776         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
3777
3778         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3779
3780         if (INTEL_GEN(dev_priv) >= 4)
3781                 dspaddr_offset = plane_state->color_plane[0].offset;
3782         else
3783                 dspaddr_offset = linear_offset;
3784
3785         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3786
3787         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3788
3789         if (INTEL_GEN(dev_priv) < 4) {
3790                 /*
3791                  * PLANE_A doesn't actually have a full window
3792                  * generator but let's assume we still need to
3793                  * program whatever is there.
3794                  */
3795                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3796                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3797                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3798         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3799                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3800                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3801                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3802                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3803         }
3804
3805         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3806                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3807         } else if (INTEL_GEN(dev_priv) >= 4) {
3808                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3809                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3810         }
3811
3812         /*
3813          * The control register self-arms if the plane was previously
3814          * disabled. Try to make the plane enable atomic by writing
3815          * the control register just before the surface register.
3816          */
3817         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3818         if (INTEL_GEN(dev_priv) >= 4)
3819                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3820                               intel_plane_ggtt_offset(plane_state) +
3821                               dspaddr_offset);
3822         else
3823                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3824                               intel_plane_ggtt_offset(plane_state) +
3825                               dspaddr_offset);
3826
3827         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3828 }
3829
3830 static void i9xx_disable_plane(struct intel_plane *plane,
3831                                const struct intel_crtc_state *crtc_state)
3832 {
3833         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3834         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3835         unsigned long irqflags;
3836         u32 dspcntr;
3837
3838         /*
3839          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
3840          * enable on ilk+ affect the pipe bottom color as
3841          * well, so we must configure them even if the plane
3842          * is disabled.
3843          *
3844          * On pre-g4x there is no way to gamma correct the
3845          * pipe bottom color but we'll keep on doing this
3846          * anyway so that the crtc state readout works correctly.
3847          */
3848         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
3849
3850         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3851
3852         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3853         if (INTEL_GEN(dev_priv) >= 4)
3854                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3855         else
3856                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3857
3858         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3859 }
3860
3861 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3862                                     enum pipe *pipe)
3863 {
3864         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3865         enum intel_display_power_domain power_domain;
3866         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3867         intel_wakeref_t wakeref;
3868         bool ret;
3869         u32 val;
3870
3871         /*
3872          * Not 100% correct for planes that can move between pipes,
3873          * but that's only the case for gen2-4 which don't have any
3874          * display power wells.
3875          */
3876         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3877         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3878         if (!wakeref)
3879                 return false;
3880
3881         val = I915_READ(DSPCNTR(i9xx_plane));
3882
3883         ret = val & DISPLAY_PLANE_ENABLE;
3884
3885         if (INTEL_GEN(dev_priv) >= 5)
3886                 *pipe = plane->pipe;
3887         else
3888                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3889                         DISPPLANE_SEL_PIPE_SHIFT;
3890
3891         intel_display_power_put(dev_priv, power_domain, wakeref);
3892
3893         return ret;
3894 }
3895
3896 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3897 {
3898         struct drm_device *dev = intel_crtc->base.dev;
3899         struct drm_i915_private *dev_priv = to_i915(dev);
3900
3901         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3902         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3903         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3904 }
3905
3906 /*
3907  * This function detaches (aka. unbinds) unused scalers in hardware
3908  */
3909 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3910 {
3911         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3912         const struct intel_crtc_scaler_state *scaler_state =
3913                 &crtc_state->scaler_state;
3914         int i;
3915
3916         /* loop through and disable scalers that aren't in use */
3917         for (i = 0; i < intel_crtc->num_scalers; i++) {
3918                 if (!scaler_state->scalers[i].in_use)
3919                         skl_detach_scaler(intel_crtc, i);
3920         }
3921 }
3922
3923 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3924                                           int color_plane, unsigned int rotation)
3925 {
3926         /*
3927          * The stride is either expressed as a multiple of 64 bytes chunks for
3928          * linear buffers or in number of tiles for tiled buffers.
3929          */
3930         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3931                 return 64;
3932         else if (drm_rotation_90_or_270(rotation))
3933                 return intel_tile_height(fb, color_plane);
3934         else
3935                 return intel_tile_width_bytes(fb, color_plane);
3936 }
3937
3938 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3939                      int color_plane)
3940 {
3941         const struct drm_framebuffer *fb = plane_state->base.fb;
3942         unsigned int rotation = plane_state->base.rotation;
3943         u32 stride = plane_state->color_plane[color_plane].stride;
3944
3945         if (color_plane >= fb->format->num_planes)
3946                 return 0;
3947
3948         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3949 }
3950
3951 static u32 skl_plane_ctl_format(u32 pixel_format)
3952 {
3953         switch (pixel_format) {
3954         case DRM_FORMAT_C8:
3955                 return PLANE_CTL_FORMAT_INDEXED;
3956         case DRM_FORMAT_RGB565:
3957                 return PLANE_CTL_FORMAT_RGB_565;
3958         case DRM_FORMAT_XBGR8888:
3959         case DRM_FORMAT_ABGR8888:
3960                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3961         case DRM_FORMAT_XRGB8888:
3962         case DRM_FORMAT_ARGB8888:
3963                 return PLANE_CTL_FORMAT_XRGB_8888;
3964         case DRM_FORMAT_XBGR2101010:
3965                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
3966         case DRM_FORMAT_XRGB2101010:
3967                 return PLANE_CTL_FORMAT_XRGB_2101010;
3968         case DRM_FORMAT_XBGR16161616F:
3969         case DRM_FORMAT_ABGR16161616F:
3970                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
3971         case DRM_FORMAT_XRGB16161616F:
3972         case DRM_FORMAT_ARGB16161616F:
3973                 return PLANE_CTL_FORMAT_XRGB_16161616F;
3974         case DRM_FORMAT_YUYV:
3975                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3976         case DRM_FORMAT_YVYU:
3977                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3978         case DRM_FORMAT_UYVY:
3979                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3980         case DRM_FORMAT_VYUY:
3981                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3982         case DRM_FORMAT_NV12:
3983                 return PLANE_CTL_FORMAT_NV12;
3984         case DRM_FORMAT_P010:
3985                 return PLANE_CTL_FORMAT_P010;
3986         case DRM_FORMAT_P012:
3987                 return PLANE_CTL_FORMAT_P012;
3988         case DRM_FORMAT_P016:
3989                 return PLANE_CTL_FORMAT_P016;
3990         case DRM_FORMAT_Y210:
3991                 return PLANE_CTL_FORMAT_Y210;
3992         case DRM_FORMAT_Y212:
3993                 return PLANE_CTL_FORMAT_Y212;
3994         case DRM_FORMAT_Y216:
3995                 return PLANE_CTL_FORMAT_Y216;
3996         case DRM_FORMAT_XVYU2101010:
3997                 return PLANE_CTL_FORMAT_Y410;
3998         case DRM_FORMAT_XVYU12_16161616:
3999                 return PLANE_CTL_FORMAT_Y412;
4000         case DRM_FORMAT_XVYU16161616:
4001                 return PLANE_CTL_FORMAT_Y416;
4002         default:
4003                 MISSING_CASE(pixel_format);
4004         }
4005
4006         return 0;
4007 }
4008
4009 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4010 {
4011         if (!plane_state->base.fb->format->has_alpha)
4012                 return PLANE_CTL_ALPHA_DISABLE;
4013
4014         switch (plane_state->base.pixel_blend_mode) {
4015         case DRM_MODE_BLEND_PIXEL_NONE:
4016                 return PLANE_CTL_ALPHA_DISABLE;
4017         case DRM_MODE_BLEND_PREMULTI:
4018                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4019         case DRM_MODE_BLEND_COVERAGE:
4020                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4021         default:
4022                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4023                 return PLANE_CTL_ALPHA_DISABLE;
4024         }
4025 }
4026
4027 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4028 {
4029         if (!plane_state->base.fb->format->has_alpha)
4030                 return PLANE_COLOR_ALPHA_DISABLE;
4031
4032         switch (plane_state->base.pixel_blend_mode) {
4033         case DRM_MODE_BLEND_PIXEL_NONE:
4034                 return PLANE_COLOR_ALPHA_DISABLE;
4035         case DRM_MODE_BLEND_PREMULTI:
4036                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4037         case DRM_MODE_BLEND_COVERAGE:
4038                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4039         default:
4040                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4041                 return PLANE_COLOR_ALPHA_DISABLE;
4042         }
4043 }
4044
4045 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4046 {
4047         switch (fb_modifier) {
4048         case DRM_FORMAT_MOD_LINEAR:
4049                 break;
4050         case I915_FORMAT_MOD_X_TILED:
4051                 return PLANE_CTL_TILED_X;
4052         case I915_FORMAT_MOD_Y_TILED:
4053                 return PLANE_CTL_TILED_Y;
4054         case I915_FORMAT_MOD_Y_TILED_CCS:
4055                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4056         case I915_FORMAT_MOD_Yf_TILED:
4057                 return PLANE_CTL_TILED_YF;
4058         case I915_FORMAT_MOD_Yf_TILED_CCS:
4059                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4060         default:
4061                 MISSING_CASE(fb_modifier);
4062         }
4063
4064         return 0;
4065 }
4066
4067 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4068 {
4069         switch (rotate) {
4070         case DRM_MODE_ROTATE_0:
4071                 break;
4072         /*
4073          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4074          * while i915 HW rotation is clockwise, thats why this swapping.
4075          */
4076         case DRM_MODE_ROTATE_90:
4077                 return PLANE_CTL_ROTATE_270;
4078         case DRM_MODE_ROTATE_180:
4079                 return PLANE_CTL_ROTATE_180;
4080         case DRM_MODE_ROTATE_270:
4081                 return PLANE_CTL_ROTATE_90;
4082         default:
4083                 MISSING_CASE(rotate);
4084         }
4085
4086         return 0;
4087 }
4088
4089 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4090 {
4091         switch (reflect) {
4092         case 0:
4093                 break;
4094         case DRM_MODE_REFLECT_X:
4095                 return PLANE_CTL_FLIP_HORIZONTAL;
4096         case DRM_MODE_REFLECT_Y:
4097         default:
4098                 MISSING_CASE(reflect);
4099         }
4100
4101         return 0;
4102 }
4103
4104 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4105 {
4106         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4107         u32 plane_ctl = 0;
4108
4109         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4110                 return plane_ctl;
4111
4112         if (crtc_state->gamma_enable)
4113                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4114
4115         if (crtc_state->csc_enable)
4116                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4117
4118         return plane_ctl;
4119 }
4120
4121 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4122                   const struct intel_plane_state *plane_state)
4123 {
4124         struct drm_i915_private *dev_priv =
4125                 to_i915(plane_state->base.plane->dev);
4126         const struct drm_framebuffer *fb = plane_state->base.fb;
4127         unsigned int rotation = plane_state->base.rotation;
4128         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4129         u32 plane_ctl;
4130
4131         plane_ctl = PLANE_CTL_ENABLE;
4132
4133         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4134                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4135                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4136
4137                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4138                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4139
4140                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4141                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4142         }
4143
4144         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4145         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4146         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4147
4148         if (INTEL_GEN(dev_priv) >= 10)
4149                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4150                                                 DRM_MODE_REFLECT_MASK);
4151
4152         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4153                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4154         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4155                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4156
4157         return plane_ctl;
4158 }
4159
4160 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4161 {
4162         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4163         u32 plane_color_ctl = 0;
4164
4165         if (INTEL_GEN(dev_priv) >= 11)
4166                 return plane_color_ctl;
4167
4168         if (crtc_state->gamma_enable)
4169                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4170
4171         if (crtc_state->csc_enable)
4172                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4173
4174         return plane_color_ctl;
4175 }
4176
4177 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4178                         const struct intel_plane_state *plane_state)
4179 {
4180         struct drm_i915_private *dev_priv =
4181                 to_i915(plane_state->base.plane->dev);
4182         const struct drm_framebuffer *fb = plane_state->base.fb;
4183         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4184         u32 plane_color_ctl = 0;
4185
4186         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4187         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4188
4189         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4190                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4191                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4192                 else
4193                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4194
4195                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4196                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4197         } else if (fb->format->is_yuv) {
4198                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4199         }
4200
4201         return plane_color_ctl;
4202 }
4203
4204 static int
4205 __intel_display_resume(struct drm_device *dev,
4206                        struct drm_atomic_state *state,
4207                        struct drm_modeset_acquire_ctx *ctx)
4208 {
4209         struct drm_crtc_state *crtc_state;
4210         struct drm_crtc *crtc;
4211         int i, ret;
4212
4213         intel_modeset_setup_hw_state(dev, ctx);
4214         i915_redisable_vga(to_i915(dev));
4215
4216         if (!state)
4217                 return 0;
4218
4219         /*
4220          * We've duplicated the state, pointers to the old state are invalid.
4221          *
4222          * Don't attempt to use the old state until we commit the duplicated state.
4223          */
4224         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4225                 /*
4226                  * Force recalculation even if we restore
4227                  * current state. With fast modeset this may not result
4228                  * in a modeset when the state is compatible.
4229                  */
4230                 crtc_state->mode_changed = true;
4231         }
4232
4233         /* ignore any reset values/BIOS leftovers in the WM registers */
4234         if (!HAS_GMCH(to_i915(dev)))
4235                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4236
4237         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4238
4239         WARN_ON(ret == -EDEADLK);
4240         return ret;
4241 }
4242
4243 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4244 {
4245         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4246                 intel_has_gpu_reset(dev_priv));
4247 }
4248
4249 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4250 {
4251         struct drm_device *dev = &dev_priv->drm;
4252         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4253         struct drm_atomic_state *state;
4254         int ret;
4255
4256         /* reset doesn't touch the display */
4257         if (!i915_modparams.force_reset_modeset_test &&
4258             !gpu_reset_clobbers_display(dev_priv))
4259                 return;
4260
4261         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4262         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4263         smp_mb__after_atomic();
4264         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4265
4266         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4267                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4268                 intel_gt_set_wedged(&dev_priv->gt);
4269         }
4270
4271         /*
4272          * Need mode_config.mutex so that we don't
4273          * trample ongoing ->detect() and whatnot.
4274          */
4275         mutex_lock(&dev->mode_config.mutex);
4276         drm_modeset_acquire_init(ctx, 0);
4277         while (1) {
4278                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4279                 if (ret != -EDEADLK)
4280                         break;
4281
4282                 drm_modeset_backoff(ctx);
4283         }
4284         /*
4285          * Disabling the crtcs gracefully seems nicer. Also the
4286          * g33 docs say we should at least disable all the planes.
4287          */
4288         state = drm_atomic_helper_duplicate_state(dev, ctx);
4289         if (IS_ERR(state)) {
4290                 ret = PTR_ERR(state);
4291                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4292                 return;
4293         }
4294
4295         ret = drm_atomic_helper_disable_all(dev, ctx);
4296         if (ret) {
4297                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4298                 drm_atomic_state_put(state);
4299                 return;
4300         }
4301
4302         dev_priv->modeset_restore_state = state;
4303         state->acquire_ctx = ctx;
4304 }
4305
4306 void intel_finish_reset(struct drm_i915_private *dev_priv)
4307 {
4308         struct drm_device *dev = &dev_priv->drm;
4309         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4310         struct drm_atomic_state *state;
4311         int ret;
4312
4313         /* reset doesn't touch the display */
4314         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4315                 return;
4316
4317         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4318         if (!state)
4319                 goto unlock;
4320
4321         /* reset doesn't touch the display */
4322         if (!gpu_reset_clobbers_display(dev_priv)) {
4323                 /* for testing only restore the display */
4324                 ret = __intel_display_resume(dev, state, ctx);
4325                 if (ret)
4326                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4327         } else {
4328                 /*
4329                  * The display has been reset as well,
4330                  * so need a full re-initialization.
4331                  */
4332                 intel_pps_unlock_regs_wa(dev_priv);
4333                 intel_modeset_init_hw(dev);
4334                 intel_init_clock_gating(dev_priv);
4335
4336                 spin_lock_irq(&dev_priv->irq_lock);
4337                 if (dev_priv->display.hpd_irq_setup)
4338                         dev_priv->display.hpd_irq_setup(dev_priv);
4339                 spin_unlock_irq(&dev_priv->irq_lock);
4340
4341                 ret = __intel_display_resume(dev, state, ctx);
4342                 if (ret)
4343                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4344
4345                 intel_hpd_init(dev_priv);
4346         }
4347
4348         drm_atomic_state_put(state);
4349 unlock:
4350         drm_modeset_drop_locks(ctx);
4351         drm_modeset_acquire_fini(ctx);
4352         mutex_unlock(&dev->mode_config.mutex);
4353
4354         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4355 }
4356
4357 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4358 {
4359         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4360         enum pipe pipe = crtc->pipe;
4361         u32 tmp;
4362
4363         tmp = I915_READ(PIPE_CHICKEN(pipe));
4364
4365         /*
4366          * Display WA #1153: icl
4367          * enable hardware to bypass the alpha math
4368          * and rounding for per-pixel values 00 and 0xff
4369          */
4370         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4371         /*
4372          * Display WA # 1605353570: icl
4373          * Set the pixel rounding bit to 1 for allowing
4374          * passthrough of Frame buffer pixels unmodified
4375          * across pipe
4376          */
4377         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4378         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4379 }
4380
4381 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
4382                                      const struct intel_crtc_state *new_crtc_state)
4383 {
4384         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
4385         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4386
4387         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
4388         crtc->base.mode = new_crtc_state->base.mode;
4389
4390         /*
4391          * Update pipe size and adjust fitter if needed: the reason for this is
4392          * that in compute_mode_changes we check the native mode (not the pfit
4393          * mode) to see if we can flip rather than do a full mode set. In the
4394          * fastboot case, we'll flip, but if we don't update the pipesrc and
4395          * pfit state, we'll end up with a big fb scanned out into the wrong
4396          * sized surface.
4397          */
4398
4399         I915_WRITE(PIPESRC(crtc->pipe),
4400                    ((new_crtc_state->pipe_src_w - 1) << 16) |
4401                    (new_crtc_state->pipe_src_h - 1));
4402
4403         /* on skylake this is done by detaching scalers */
4404         if (INTEL_GEN(dev_priv) >= 9) {
4405                 skl_detach_scalers(new_crtc_state);
4406
4407                 if (new_crtc_state->pch_pfit.enabled)
4408                         skylake_pfit_enable(new_crtc_state);
4409         } else if (HAS_PCH_SPLIT(dev_priv)) {
4410                 if (new_crtc_state->pch_pfit.enabled)
4411                         ironlake_pfit_enable(new_crtc_state);
4412                 else if (old_crtc_state->pch_pfit.enabled)
4413                         ironlake_pfit_disable(old_crtc_state);
4414         }
4415
4416         if (INTEL_GEN(dev_priv) >= 11)
4417                 icl_set_pipe_chicken(crtc);
4418 }
4419
4420 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4421 {
4422         struct drm_device *dev = crtc->base.dev;
4423         struct drm_i915_private *dev_priv = to_i915(dev);
4424         enum pipe pipe = crtc->pipe;
4425         i915_reg_t reg;
4426         u32 temp;
4427
4428         /* enable normal train */
4429         reg = FDI_TX_CTL(pipe);
4430         temp = I915_READ(reg);
4431         if (IS_IVYBRIDGE(dev_priv)) {
4432                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4433                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4434         } else {
4435                 temp &= ~FDI_LINK_TRAIN_NONE;
4436                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4437         }
4438         I915_WRITE(reg, temp);
4439
4440         reg = FDI_RX_CTL(pipe);
4441         temp = I915_READ(reg);
4442         if (HAS_PCH_CPT(dev_priv)) {
4443                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4444                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4445         } else {
4446                 temp &= ~FDI_LINK_TRAIN_NONE;
4447                 temp |= FDI_LINK_TRAIN_NONE;
4448         }
4449         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4450
4451         /* wait one idle pattern time */
4452         POSTING_READ(reg);
4453         udelay(1000);
4454
4455         /* IVB wants error correction enabled */
4456         if (IS_IVYBRIDGE(dev_priv))
4457                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4458                            FDI_FE_ERRC_ENABLE);
4459 }
4460
4461 /* The FDI link training functions for ILK/Ibexpeak. */
4462 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4463                                     const struct intel_crtc_state *crtc_state)
4464 {
4465         struct drm_device *dev = crtc->base.dev;
4466         struct drm_i915_private *dev_priv = to_i915(dev);
4467         enum pipe pipe = crtc->pipe;
4468         i915_reg_t reg;
4469         u32 temp, tries;
4470
4471         /* FDI needs bits from pipe first */
4472         assert_pipe_enabled(dev_priv, pipe);
4473
4474         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4475            for train result */
4476         reg = FDI_RX_IMR(pipe);
4477         temp = I915_READ(reg);
4478         temp &= ~FDI_RX_SYMBOL_LOCK;
4479         temp &= ~FDI_RX_BIT_LOCK;
4480         I915_WRITE(reg, temp);
4481         I915_READ(reg);
4482         udelay(150);
4483
4484         /* enable CPU FDI TX and PCH FDI RX */
4485         reg = FDI_TX_CTL(pipe);
4486         temp = I915_READ(reg);
4487         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4488         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4489         temp &= ~FDI_LINK_TRAIN_NONE;
4490         temp |= FDI_LINK_TRAIN_PATTERN_1;
4491         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4492
4493         reg = FDI_RX_CTL(pipe);
4494         temp = I915_READ(reg);
4495         temp &= ~FDI_LINK_TRAIN_NONE;
4496         temp |= FDI_LINK_TRAIN_PATTERN_1;
4497         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4498
4499         POSTING_READ(reg);
4500         udelay(150);
4501
4502         /* Ironlake workaround, enable clock pointer after FDI enable*/
4503         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4504         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4505                    FDI_RX_PHASE_SYNC_POINTER_EN);
4506
4507         reg = FDI_RX_IIR(pipe);
4508         for (tries = 0; tries < 5; tries++) {
4509                 temp = I915_READ(reg);
4510                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4511
4512                 if ((temp & FDI_RX_BIT_LOCK)) {
4513                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4514                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4515                         break;
4516                 }
4517         }
4518         if (tries == 5)
4519                 DRM_ERROR("FDI train 1 fail!\n");
4520
4521         /* Train 2 */
4522         reg = FDI_TX_CTL(pipe);
4523         temp = I915_READ(reg);
4524         temp &= ~FDI_LINK_TRAIN_NONE;
4525         temp |= FDI_LINK_TRAIN_PATTERN_2;
4526         I915_WRITE(reg, temp);
4527
4528         reg = FDI_RX_CTL(pipe);
4529         temp = I915_READ(reg);
4530         temp &= ~FDI_LINK_TRAIN_NONE;
4531         temp |= FDI_LINK_TRAIN_PATTERN_2;
4532         I915_WRITE(reg, temp);
4533
4534         POSTING_READ(reg);
4535         udelay(150);
4536
4537         reg = FDI_RX_IIR(pipe);
4538         for (tries = 0; tries < 5; tries++) {
4539                 temp = I915_READ(reg);
4540                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4541
4542                 if (temp & FDI_RX_SYMBOL_LOCK) {
4543                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4544                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4545                         break;
4546                 }
4547         }
4548         if (tries == 5)
4549                 DRM_ERROR("FDI train 2 fail!\n");
4550
4551         DRM_DEBUG_KMS("FDI train done\n");
4552
4553 }
4554
4555 static const int snb_b_fdi_train_param[] = {
4556         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4557         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4558         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4559         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4560 };
4561
4562 /* The FDI link training functions for SNB/Cougarpoint. */
4563 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4564                                 const struct intel_crtc_state *crtc_state)
4565 {
4566         struct drm_device *dev = crtc->base.dev;
4567         struct drm_i915_private *dev_priv = to_i915(dev);
4568         enum pipe pipe = crtc->pipe;
4569         i915_reg_t reg;
4570         u32 temp, i, retry;
4571
4572         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4573            for train result */
4574         reg = FDI_RX_IMR(pipe);
4575         temp = I915_READ(reg);
4576         temp &= ~FDI_RX_SYMBOL_LOCK;
4577         temp &= ~FDI_RX_BIT_LOCK;
4578         I915_WRITE(reg, temp);
4579
4580         POSTING_READ(reg);
4581         udelay(150);
4582
4583         /* enable CPU FDI TX and PCH FDI RX */
4584         reg = FDI_TX_CTL(pipe);
4585         temp = I915_READ(reg);
4586         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4587         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4588         temp &= ~FDI_LINK_TRAIN_NONE;
4589         temp |= FDI_LINK_TRAIN_PATTERN_1;
4590         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4591         /* SNB-B */
4592         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4593         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4594
4595         I915_WRITE(FDI_RX_MISC(pipe),
4596                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4597
4598         reg = FDI_RX_CTL(pipe);
4599         temp = I915_READ(reg);
4600         if (HAS_PCH_CPT(dev_priv)) {
4601                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4602                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4603         } else {
4604                 temp &= ~FDI_LINK_TRAIN_NONE;
4605                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4606         }
4607         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4608
4609         POSTING_READ(reg);
4610         udelay(150);
4611
4612         for (i = 0; i < 4; i++) {
4613                 reg = FDI_TX_CTL(pipe);
4614                 temp = I915_READ(reg);
4615                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4616                 temp |= snb_b_fdi_train_param[i];
4617                 I915_WRITE(reg, temp);
4618
4619                 POSTING_READ(reg);
4620                 udelay(500);
4621
4622                 for (retry = 0; retry < 5; retry++) {
4623                         reg = FDI_RX_IIR(pipe);
4624                         temp = I915_READ(reg);
4625                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4626                         if (temp & FDI_RX_BIT_LOCK) {
4627                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4628                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4629                                 break;
4630                         }
4631                         udelay(50);
4632                 }
4633                 if (retry < 5)
4634                         break;
4635         }
4636         if (i == 4)
4637                 DRM_ERROR("FDI train 1 fail!\n");
4638
4639         /* Train 2 */
4640         reg = FDI_TX_CTL(pipe);
4641         temp = I915_READ(reg);
4642         temp &= ~FDI_LINK_TRAIN_NONE;
4643         temp |= FDI_LINK_TRAIN_PATTERN_2;
4644         if (IS_GEN(dev_priv, 6)) {
4645                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4646                 /* SNB-B */
4647                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4648         }
4649         I915_WRITE(reg, temp);
4650
4651         reg = FDI_RX_CTL(pipe);
4652         temp = I915_READ(reg);
4653         if (HAS_PCH_CPT(dev_priv)) {
4654                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4655                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4656         } else {
4657                 temp &= ~FDI_LINK_TRAIN_NONE;
4658                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4659         }
4660         I915_WRITE(reg, temp);
4661
4662         POSTING_READ(reg);
4663         udelay(150);
4664
4665         for (i = 0; i < 4; i++) {
4666                 reg = FDI_TX_CTL(pipe);
4667                 temp = I915_READ(reg);
4668                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4669                 temp |= snb_b_fdi_train_param[i];
4670                 I915_WRITE(reg, temp);
4671
4672                 POSTING_READ(reg);
4673                 udelay(500);
4674
4675                 for (retry = 0; retry < 5; retry++) {
4676                         reg = FDI_RX_IIR(pipe);
4677                         temp = I915_READ(reg);
4678                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4679                         if (temp & FDI_RX_SYMBOL_LOCK) {
4680                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4681                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4682                                 break;
4683                         }
4684                         udelay(50);
4685                 }
4686                 if (retry < 5)
4687                         break;
4688         }
4689         if (i == 4)
4690                 DRM_ERROR("FDI train 2 fail!\n");
4691
4692         DRM_DEBUG_KMS("FDI train done.\n");
4693 }
4694
4695 /* Manual link training for Ivy Bridge A0 parts */
4696 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4697                                       const struct intel_crtc_state *crtc_state)
4698 {
4699         struct drm_device *dev = crtc->base.dev;
4700         struct drm_i915_private *dev_priv = to_i915(dev);
4701         enum pipe pipe = crtc->pipe;
4702         i915_reg_t reg;
4703         u32 temp, i, j;
4704
4705         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4706            for train result */
4707         reg = FDI_RX_IMR(pipe);
4708         temp = I915_READ(reg);
4709         temp &= ~FDI_RX_SYMBOL_LOCK;
4710         temp &= ~FDI_RX_BIT_LOCK;
4711         I915_WRITE(reg, temp);
4712
4713         POSTING_READ(reg);
4714         udelay(150);
4715
4716         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4717                       I915_READ(FDI_RX_IIR(pipe)));
4718
4719         /* Try each vswing and preemphasis setting twice before moving on */
4720         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4721                 /* disable first in case we need to retry */
4722                 reg = FDI_TX_CTL(pipe);
4723                 temp = I915_READ(reg);
4724                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4725                 temp &= ~FDI_TX_ENABLE;
4726                 I915_WRITE(reg, temp);
4727
4728                 reg = FDI_RX_CTL(pipe);
4729                 temp = I915_READ(reg);
4730                 temp &= ~FDI_LINK_TRAIN_AUTO;
4731                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4732                 temp &= ~FDI_RX_ENABLE;
4733                 I915_WRITE(reg, temp);
4734
4735                 /* enable CPU FDI TX and PCH FDI RX */
4736                 reg = FDI_TX_CTL(pipe);
4737                 temp = I915_READ(reg);
4738                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4739                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4740                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4741                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4742                 temp |= snb_b_fdi_train_param[j/2];
4743                 temp |= FDI_COMPOSITE_SYNC;
4744                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4745
4746                 I915_WRITE(FDI_RX_MISC(pipe),
4747                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4748
4749                 reg = FDI_RX_CTL(pipe);
4750                 temp = I915_READ(reg);
4751                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4752                 temp |= FDI_COMPOSITE_SYNC;
4753                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4754
4755                 POSTING_READ(reg);
4756                 udelay(1); /* should be 0.5us */
4757
4758                 for (i = 0; i < 4; i++) {
4759                         reg = FDI_RX_IIR(pipe);
4760                         temp = I915_READ(reg);
4761                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4762
4763                         if (temp & FDI_RX_BIT_LOCK ||
4764                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4765                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4766                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4767                                               i);
4768                                 break;
4769                         }
4770                         udelay(1); /* should be 0.5us */
4771                 }
4772                 if (i == 4) {
4773                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4774                         continue;
4775                 }
4776
4777                 /* Train 2 */
4778                 reg = FDI_TX_CTL(pipe);
4779                 temp = I915_READ(reg);
4780                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4781                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4782                 I915_WRITE(reg, temp);
4783
4784                 reg = FDI_RX_CTL(pipe);
4785                 temp = I915_READ(reg);
4786                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4787                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4788                 I915_WRITE(reg, temp);
4789
4790                 POSTING_READ(reg);
4791                 udelay(2); /* should be 1.5us */
4792
4793                 for (i = 0; i < 4; i++) {
4794                         reg = FDI_RX_IIR(pipe);
4795                         temp = I915_READ(reg);
4796                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4797
4798                         if (temp & FDI_RX_SYMBOL_LOCK ||
4799                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4800                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4801                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4802                                               i);
4803                                 goto train_done;
4804                         }
4805                         udelay(2); /* should be 1.5us */
4806                 }
4807                 if (i == 4)
4808                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4809         }
4810
4811 train_done:
4812         DRM_DEBUG_KMS("FDI train done.\n");
4813 }
4814
4815 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4816 {
4817         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4818         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4819         enum pipe pipe = intel_crtc->pipe;
4820         i915_reg_t reg;
4821         u32 temp;
4822
4823         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4824         reg = FDI_RX_CTL(pipe);
4825         temp = I915_READ(reg);
4826         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4827         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4828         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4829         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4830
4831         POSTING_READ(reg);
4832         udelay(200);
4833
4834         /* Switch from Rawclk to PCDclk */
4835         temp = I915_READ(reg);
4836         I915_WRITE(reg, temp | FDI_PCDCLK);
4837
4838         POSTING_READ(reg);
4839         udelay(200);
4840
4841         /* Enable CPU FDI TX PLL, always on for Ironlake */
4842         reg = FDI_TX_CTL(pipe);
4843         temp = I915_READ(reg);
4844         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4845                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4846
4847                 POSTING_READ(reg);
4848                 udelay(100);
4849         }
4850 }
4851
4852 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4853 {
4854         struct drm_device *dev = intel_crtc->base.dev;
4855         struct drm_i915_private *dev_priv = to_i915(dev);
4856         enum pipe pipe = intel_crtc->pipe;
4857         i915_reg_t reg;
4858         u32 temp;
4859
4860         /* Switch from PCDclk to Rawclk */
4861         reg = FDI_RX_CTL(pipe);
4862         temp = I915_READ(reg);
4863         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4864
4865         /* Disable CPU FDI TX PLL */
4866         reg = FDI_TX_CTL(pipe);
4867         temp = I915_READ(reg);
4868         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4869
4870         POSTING_READ(reg);
4871         udelay(100);
4872
4873         reg = FDI_RX_CTL(pipe);
4874         temp = I915_READ(reg);
4875         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4876
4877         /* Wait for the clocks to turn off. */
4878         POSTING_READ(reg);
4879         udelay(100);
4880 }
4881
4882 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4883 {
4884         struct drm_device *dev = crtc->dev;
4885         struct drm_i915_private *dev_priv = to_i915(dev);
4886         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4887         enum pipe pipe = intel_crtc->pipe;
4888         i915_reg_t reg;
4889         u32 temp;
4890
4891         /* disable CPU FDI tx and PCH FDI rx */
4892         reg = FDI_TX_CTL(pipe);
4893         temp = I915_READ(reg);
4894         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4895         POSTING_READ(reg);
4896
4897         reg = FDI_RX_CTL(pipe);
4898         temp = I915_READ(reg);
4899         temp &= ~(0x7 << 16);
4900         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4901         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4902
4903         POSTING_READ(reg);
4904         udelay(100);
4905
4906         /* Ironlake workaround, disable clock pointer after downing FDI */
4907         if (HAS_PCH_IBX(dev_priv))
4908                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4909
4910         /* still set train pattern 1 */
4911         reg = FDI_TX_CTL(pipe);
4912         temp = I915_READ(reg);
4913         temp &= ~FDI_LINK_TRAIN_NONE;
4914         temp |= FDI_LINK_TRAIN_PATTERN_1;
4915         I915_WRITE(reg, temp);
4916
4917         reg = FDI_RX_CTL(pipe);
4918         temp = I915_READ(reg);
4919         if (HAS_PCH_CPT(dev_priv)) {
4920                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4921                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4922         } else {
4923                 temp &= ~FDI_LINK_TRAIN_NONE;
4924                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4925         }
4926         /* BPC in FDI rx is consistent with that in PIPECONF */
4927         temp &= ~(0x07 << 16);
4928         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4929         I915_WRITE(reg, temp);
4930
4931         POSTING_READ(reg);
4932         udelay(100);
4933 }
4934
4935 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4936 {
4937         struct drm_crtc *crtc;
4938         bool cleanup_done;
4939
4940         drm_for_each_crtc(crtc, &dev_priv->drm) {
4941                 struct drm_crtc_commit *commit;
4942                 spin_lock(&crtc->commit_lock);
4943                 commit = list_first_entry_or_null(&crtc->commit_list,
4944                                                   struct drm_crtc_commit, commit_entry);
4945                 cleanup_done = commit ?
4946                         try_wait_for_completion(&commit->cleanup_done) : true;
4947                 spin_unlock(&crtc->commit_lock);
4948
4949                 if (cleanup_done)
4950                         continue;
4951
4952                 drm_crtc_wait_one_vblank(crtc);
4953
4954                 return true;
4955         }
4956
4957         return false;
4958 }
4959
4960 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4961 {
4962         u32 temp;
4963
4964         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4965
4966         mutex_lock(&dev_priv->sb_lock);
4967
4968         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4969         temp |= SBI_SSCCTL_DISABLE;
4970         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4971
4972         mutex_unlock(&dev_priv->sb_lock);
4973 }
4974
4975 /* Program iCLKIP clock to the desired frequency */
4976 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4977 {
4978         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4979         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4980         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4981         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4982         u32 temp;
4983
4984         lpt_disable_iclkip(dev_priv);
4985
4986         /* The iCLK virtual clock root frequency is in MHz,
4987          * but the adjusted_mode->crtc_clock in in KHz. To get the
4988          * divisors, it is necessary to divide one by another, so we
4989          * convert the virtual clock precision to KHz here for higher
4990          * precision.
4991          */
4992         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
4993                 u32 iclk_virtual_root_freq = 172800 * 1000;
4994                 u32 iclk_pi_range = 64;
4995                 u32 desired_divisor;
4996
4997                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4998                                                     clock << auxdiv);
4999                 divsel = (desired_divisor / iclk_pi_range) - 2;
5000                 phaseinc = desired_divisor % iclk_pi_range;
5001
5002                 /*
5003                  * Near 20MHz is a corner case which is
5004                  * out of range for the 7-bit divisor
5005                  */
5006                 if (divsel <= 0x7f)
5007                         break;
5008         }
5009
5010         /* This should not happen with any sane values */
5011         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5012                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5013         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5014                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5015
5016         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5017                         clock,
5018                         auxdiv,
5019                         divsel,
5020                         phasedir,
5021                         phaseinc);
5022
5023         mutex_lock(&dev_priv->sb_lock);
5024
5025         /* Program SSCDIVINTPHASE6 */
5026         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5027         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5028         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5029         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5030         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5031         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5032         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5033         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5034
5035         /* Program SSCAUXDIV */
5036         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5037         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5038         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5039         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5040
5041         /* Enable modulator and associated divider */
5042         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5043         temp &= ~SBI_SSCCTL_DISABLE;
5044         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5045
5046         mutex_unlock(&dev_priv->sb_lock);
5047
5048         /* Wait for initialization time */
5049         udelay(24);
5050
5051         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5052 }
5053
5054 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5055 {
5056         u32 divsel, phaseinc, auxdiv;
5057         u32 iclk_virtual_root_freq = 172800 * 1000;
5058         u32 iclk_pi_range = 64;
5059         u32 desired_divisor;
5060         u32 temp;
5061
5062         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5063                 return 0;
5064
5065         mutex_lock(&dev_priv->sb_lock);
5066
5067         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5068         if (temp & SBI_SSCCTL_DISABLE) {
5069                 mutex_unlock(&dev_priv->sb_lock);
5070                 return 0;
5071         }
5072
5073         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5074         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5075                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5076         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5077                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5078
5079         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5080         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5081                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5082
5083         mutex_unlock(&dev_priv->sb_lock);
5084
5085         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5086
5087         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5088                                  desired_divisor << auxdiv);
5089 }
5090
5091 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5092                                                 enum pipe pch_transcoder)
5093 {
5094         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5095         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5096         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5097
5098         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5099                    I915_READ(HTOTAL(cpu_transcoder)));
5100         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5101                    I915_READ(HBLANK(cpu_transcoder)));
5102         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5103                    I915_READ(HSYNC(cpu_transcoder)));
5104
5105         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5106                    I915_READ(VTOTAL(cpu_transcoder)));
5107         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5108                    I915_READ(VBLANK(cpu_transcoder)));
5109         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5110                    I915_READ(VSYNC(cpu_transcoder)));
5111         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5112                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5113 }
5114
5115 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5116 {
5117         u32 temp;
5118
5119         temp = I915_READ(SOUTH_CHICKEN1);
5120         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5121                 return;
5122
5123         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5124         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5125
5126         temp &= ~FDI_BC_BIFURCATION_SELECT;
5127         if (enable)
5128                 temp |= FDI_BC_BIFURCATION_SELECT;
5129
5130         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5131         I915_WRITE(SOUTH_CHICKEN1, temp);
5132         POSTING_READ(SOUTH_CHICKEN1);
5133 }
5134
5135 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5136 {
5137         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5138         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5139
5140         switch (crtc->pipe) {
5141         case PIPE_A:
5142                 break;
5143         case PIPE_B:
5144                 if (crtc_state->fdi_lanes > 2)
5145                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5146                 else
5147                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5148
5149                 break;
5150         case PIPE_C:
5151                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5152
5153                 break;
5154         default:
5155                 BUG();
5156         }
5157 }
5158
5159 /*
5160  * Finds the encoder associated with the given CRTC. This can only be
5161  * used when we know that the CRTC isn't feeding multiple encoders!
5162  */
5163 static struct intel_encoder *
5164 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5165                            const struct intel_crtc_state *crtc_state)
5166 {
5167         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5168         const struct drm_connector_state *connector_state;
5169         const struct drm_connector *connector;
5170         struct intel_encoder *encoder = NULL;
5171         int num_encoders = 0;
5172         int i;
5173
5174         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5175                 if (connector_state->crtc != &crtc->base)
5176                         continue;
5177
5178                 encoder = to_intel_encoder(connector_state->best_encoder);
5179                 num_encoders++;
5180         }
5181
5182         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5183              num_encoders, pipe_name(crtc->pipe));
5184
5185         return encoder;
5186 }
5187
5188 /*
5189  * Enable PCH resources required for PCH ports:
5190  *   - PCH PLLs
5191  *   - FDI training & RX/TX
5192  *   - update transcoder timings
5193  *   - DP transcoding bits
5194  *   - transcoder
5195  */
5196 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5197                                 const struct intel_crtc_state *crtc_state)
5198 {
5199         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5200         struct drm_device *dev = crtc->base.dev;
5201         struct drm_i915_private *dev_priv = to_i915(dev);
5202         enum pipe pipe = crtc->pipe;
5203         u32 temp;
5204
5205         assert_pch_transcoder_disabled(dev_priv, pipe);
5206
5207         if (IS_IVYBRIDGE(dev_priv))
5208                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5209
5210         /* Write the TU size bits before fdi link training, so that error
5211          * detection works. */
5212         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5213                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5214
5215         /* For PCH output, training FDI link */
5216         dev_priv->display.fdi_link_train(crtc, crtc_state);
5217
5218         /* We need to program the right clock selection before writing the pixel
5219          * mutliplier into the DPLL. */
5220         if (HAS_PCH_CPT(dev_priv)) {
5221                 u32 sel;
5222
5223                 temp = I915_READ(PCH_DPLL_SEL);
5224                 temp |= TRANS_DPLL_ENABLE(pipe);
5225                 sel = TRANS_DPLLB_SEL(pipe);
5226                 if (crtc_state->shared_dpll ==
5227                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5228                         temp |= sel;
5229                 else
5230                         temp &= ~sel;
5231                 I915_WRITE(PCH_DPLL_SEL, temp);
5232         }
5233
5234         /* XXX: pch pll's can be enabled any time before we enable the PCH
5235          * transcoder, and we actually should do this to not upset any PCH
5236          * transcoder that already use the clock when we share it.
5237          *
5238          * Note that enable_shared_dpll tries to do the right thing, but
5239          * get_shared_dpll unconditionally resets the pll - we need that to have
5240          * the right LVDS enable sequence. */
5241         intel_enable_shared_dpll(crtc_state);
5242
5243         /* set transcoder timing, panel must allow it */
5244         assert_panel_unlocked(dev_priv, pipe);
5245         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5246
5247         intel_fdi_normal_train(crtc);
5248
5249         /* For PCH DP, enable TRANS_DP_CTL */
5250         if (HAS_PCH_CPT(dev_priv) &&
5251             intel_crtc_has_dp_encoder(crtc_state)) {
5252                 const struct drm_display_mode *adjusted_mode =
5253                         &crtc_state->base.adjusted_mode;
5254                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5255                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5256                 enum port port;
5257
5258                 temp = I915_READ(reg);
5259                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5260                           TRANS_DP_SYNC_MASK |
5261                           TRANS_DP_BPC_MASK);
5262                 temp |= TRANS_DP_OUTPUT_ENABLE;
5263                 temp |= bpc << 9; /* same format but at 11:9 */
5264
5265                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5266                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5267                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5268                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5269
5270                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5271                 WARN_ON(port < PORT_B || port > PORT_D);
5272                 temp |= TRANS_DP_PORT_SEL(port);
5273
5274                 I915_WRITE(reg, temp);
5275         }
5276
5277         ironlake_enable_pch_transcoder(crtc_state);
5278 }
5279
5280 static void lpt_pch_enable(const struct intel_atomic_state *state,
5281                            const struct intel_crtc_state *crtc_state)
5282 {
5283         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5284         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5285         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5286
5287         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5288
5289         lpt_program_iclkip(crtc_state);
5290
5291         /* Set transcoder timing. */
5292         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5293
5294         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5295 }
5296
5297 static void cpt_verify_modeset(struct drm_device *dev, enum pipe pipe)
5298 {
5299         struct drm_i915_private *dev_priv = to_i915(dev);
5300         i915_reg_t dslreg = PIPEDSL(pipe);
5301         u32 temp;
5302
5303         temp = I915_READ(dslreg);
5304         udelay(500);
5305         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5306                 if (wait_for(I915_READ(dslreg) != temp, 5))
5307                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5308         }
5309 }
5310
5311 /*
5312  * The hardware phase 0.0 refers to the center of the pixel.
5313  * We want to start from the top/left edge which is phase
5314  * -0.5. That matches how the hardware calculates the scaling
5315  * factors (from top-left of the first pixel to bottom-right
5316  * of the last pixel, as opposed to the pixel centers).
5317  *
5318  * For 4:2:0 subsampled chroma planes we obviously have to
5319  * adjust that so that the chroma sample position lands in
5320  * the right spot.
5321  *
5322  * Note that for packed YCbCr 4:2:2 formats there is no way to
5323  * control chroma siting. The hardware simply replicates the
5324  * chroma samples for both of the luma samples, and thus we don't
5325  * actually get the expected MPEG2 chroma siting convention :(
5326  * The same behaviour is observed on pre-SKL platforms as well.
5327  *
5328  * Theory behind the formula (note that we ignore sub-pixel
5329  * source coordinates):
5330  * s = source sample position
5331  * d = destination sample position
5332  *
5333  * Downscaling 4:1:
5334  * -0.5
5335  * | 0.0
5336  * | |     1.5 (initial phase)
5337  * | |     |
5338  * v v     v
5339  * | s | s | s | s |
5340  * |       d       |
5341  *
5342  * Upscaling 1:4:
5343  * -0.5
5344  * | -0.375 (initial phase)
5345  * | |     0.0
5346  * | |     |
5347  * v v     v
5348  * |       s       |
5349  * | d | d | d | d |
5350  */
5351 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5352 {
5353         int phase = -0x8000;
5354         u16 trip = 0;
5355
5356         if (chroma_cosited)
5357                 phase += (sub - 1) * 0x8000 / sub;
5358
5359         phase += scale / (2 * sub);
5360
5361         /*
5362          * Hardware initial phase limited to [-0.5:1.5].
5363          * Since the max hardware scale factor is 3.0, we
5364          * should never actually excdeed 1.0 here.
5365          */
5366         WARN_ON(phase < -0x8000 || phase > 0x18000);
5367
5368         if (phase < 0)
5369                 phase = 0x10000 + phase;
5370         else
5371                 trip = PS_PHASE_TRIP;
5372
5373         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5374 }
5375
5376 #define SKL_MIN_SRC_W 8
5377 #define SKL_MAX_SRC_W 4096
5378 #define SKL_MIN_SRC_H 8
5379 #define SKL_MAX_SRC_H 4096
5380 #define SKL_MIN_DST_W 8
5381 #define SKL_MAX_DST_W 4096
5382 #define SKL_MIN_DST_H 8
5383 #define SKL_MAX_DST_H 4096
5384 #define ICL_MAX_SRC_W 5120
5385 #define ICL_MAX_SRC_H 4096
5386 #define ICL_MAX_DST_W 5120
5387 #define ICL_MAX_DST_H 4096
5388 #define SKL_MIN_YUV_420_SRC_W 16
5389 #define SKL_MIN_YUV_420_SRC_H 16
5390
5391 static int
5392 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5393                   unsigned int scaler_user, int *scaler_id,
5394                   int src_w, int src_h, int dst_w, int dst_h,
5395                   const struct drm_format_info *format, bool need_scaler)
5396 {
5397         struct intel_crtc_scaler_state *scaler_state =
5398                 &crtc_state->scaler_state;
5399         struct intel_crtc *intel_crtc =
5400                 to_intel_crtc(crtc_state->base.crtc);
5401         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5402         const struct drm_display_mode *adjusted_mode =
5403                 &crtc_state->base.adjusted_mode;
5404
5405         /*
5406          * Src coordinates are already rotated by 270 degrees for
5407          * the 90/270 degree plane rotation cases (to match the
5408          * GTT mapping), hence no need to account for rotation here.
5409          */
5410         if (src_w != dst_w || src_h != dst_h)
5411                 need_scaler = true;
5412
5413         /*
5414          * Scaling/fitting not supported in IF-ID mode in GEN9+
5415          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5416          * Once NV12 is enabled, handle it here while allocating scaler
5417          * for NV12.
5418          */
5419         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
5420             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5421                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5422                 return -EINVAL;
5423         }
5424
5425         /*
5426          * if plane is being disabled or scaler is no more required or force detach
5427          *  - free scaler binded to this plane/crtc
5428          *  - in order to do this, update crtc->scaler_usage
5429          *
5430          * Here scaler state in crtc_state is set free so that
5431          * scaler can be assigned to other user. Actual register
5432          * update to free the scaler is done in plane/panel-fit programming.
5433          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5434          */
5435         if (force_detach || !need_scaler) {
5436                 if (*scaler_id >= 0) {
5437                         scaler_state->scaler_users &= ~(1 << scaler_user);
5438                         scaler_state->scalers[*scaler_id].in_use = 0;
5439
5440                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5441                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5442                                 intel_crtc->pipe, scaler_user, *scaler_id,
5443                                 scaler_state->scaler_users);
5444                         *scaler_id = -1;
5445                 }
5446                 return 0;
5447         }
5448
5449         if (format && is_planar_yuv_format(format->format) &&
5450             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5451                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5452                 return -EINVAL;
5453         }
5454
5455         /* range checks */
5456         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5457             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5458             (INTEL_GEN(dev_priv) >= 11 &&
5459              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5460               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5461             (INTEL_GEN(dev_priv) < 11 &&
5462              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5463               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5464                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5465                         "size is out of scaler range\n",
5466                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5467                 return -EINVAL;
5468         }
5469
5470         /* mark this plane as a scaler user in crtc_state */
5471         scaler_state->scaler_users |= (1 << scaler_user);
5472         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5473                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5474                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5475                 scaler_state->scaler_users);
5476
5477         return 0;
5478 }
5479
5480 /**
5481  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5482  *
5483  * @state: crtc's scaler state
5484  *
5485  * Return
5486  *     0 - scaler_usage updated successfully
5487  *    error - requested scaling cannot be supported or other error condition
5488  */
5489 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5490 {
5491         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
5492         bool need_scaler = false;
5493
5494         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5495                 need_scaler = true;
5496
5497         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
5498                                  &state->scaler_state.scaler_id,
5499                                  state->pipe_src_w, state->pipe_src_h,
5500                                  adjusted_mode->crtc_hdisplay,
5501                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
5502 }
5503
5504 /**
5505  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5506  * @crtc_state: crtc's scaler state
5507  * @plane_state: atomic plane state to update
5508  *
5509  * Return
5510  *     0 - scaler_usage updated successfully
5511  *    error - requested scaling cannot be supported or other error condition
5512  */
5513 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5514                                    struct intel_plane_state *plane_state)
5515 {
5516         struct intel_plane *intel_plane =
5517                 to_intel_plane(plane_state->base.plane);
5518         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5519         struct drm_framebuffer *fb = plane_state->base.fb;
5520         int ret;
5521         bool force_detach = !fb || !plane_state->base.visible;
5522         bool need_scaler = false;
5523
5524         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5525         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5526             fb && is_planar_yuv_format(fb->format->format))
5527                 need_scaler = true;
5528
5529         ret = skl_update_scaler(crtc_state, force_detach,
5530                                 drm_plane_index(&intel_plane->base),
5531                                 &plane_state->scaler_id,
5532                                 drm_rect_width(&plane_state->base.src) >> 16,
5533                                 drm_rect_height(&plane_state->base.src) >> 16,
5534                                 drm_rect_width(&plane_state->base.dst),
5535                                 drm_rect_height(&plane_state->base.dst),
5536                                 fb ? fb->format : NULL, need_scaler);
5537
5538         if (ret || plane_state->scaler_id < 0)
5539                 return ret;
5540
5541         /* check colorkey */
5542         if (plane_state->ckey.flags) {
5543                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5544                               intel_plane->base.base.id,
5545                               intel_plane->base.name);
5546                 return -EINVAL;
5547         }
5548
5549         /* Check src format */
5550         switch (fb->format->format) {
5551         case DRM_FORMAT_RGB565:
5552         case DRM_FORMAT_XBGR8888:
5553         case DRM_FORMAT_XRGB8888:
5554         case DRM_FORMAT_ABGR8888:
5555         case DRM_FORMAT_ARGB8888:
5556         case DRM_FORMAT_XRGB2101010:
5557         case DRM_FORMAT_XBGR2101010:
5558         case DRM_FORMAT_XBGR16161616F:
5559         case DRM_FORMAT_ABGR16161616F:
5560         case DRM_FORMAT_XRGB16161616F:
5561         case DRM_FORMAT_ARGB16161616F:
5562         case DRM_FORMAT_YUYV:
5563         case DRM_FORMAT_YVYU:
5564         case DRM_FORMAT_UYVY:
5565         case DRM_FORMAT_VYUY:
5566         case DRM_FORMAT_NV12:
5567         case DRM_FORMAT_P010:
5568         case DRM_FORMAT_P012:
5569         case DRM_FORMAT_P016:
5570         case DRM_FORMAT_Y210:
5571         case DRM_FORMAT_Y212:
5572         case DRM_FORMAT_Y216:
5573         case DRM_FORMAT_XVYU2101010:
5574         case DRM_FORMAT_XVYU12_16161616:
5575         case DRM_FORMAT_XVYU16161616:
5576                 break;
5577         default:
5578                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5579                               intel_plane->base.base.id, intel_plane->base.name,
5580                               fb->base.id, fb->format->format);
5581                 return -EINVAL;
5582         }
5583
5584         return 0;
5585 }
5586
5587 static void skylake_scaler_disable(struct intel_crtc *crtc)
5588 {
5589         int i;
5590
5591         for (i = 0; i < crtc->num_scalers; i++)
5592                 skl_detach_scaler(crtc, i);
5593 }
5594
5595 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5596 {
5597         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5598         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5599         enum pipe pipe = crtc->pipe;
5600         const struct intel_crtc_scaler_state *scaler_state =
5601                 &crtc_state->scaler_state;
5602
5603         if (crtc_state->pch_pfit.enabled) {
5604                 u16 uv_rgb_hphase, uv_rgb_vphase;
5605                 int pfit_w, pfit_h, hscale, vscale;
5606                 int id;
5607
5608                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5609                         return;
5610
5611                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5612                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5613
5614                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5615                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5616
5617                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5618                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5619
5620                 id = scaler_state->scaler_id;
5621                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5622                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5623                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5624                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5625                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5626                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5627                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5628                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5629         }
5630 }
5631
5632 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5633 {
5634         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5635         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5636         enum pipe pipe = crtc->pipe;
5637
5638         if (crtc_state->pch_pfit.enabled) {
5639                 /* Force use of hard-coded filter coefficients
5640                  * as some pre-programmed values are broken,
5641                  * e.g. x201.
5642                  */
5643                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5644                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5645                                                  PF_PIPE_SEL_IVB(pipe));
5646                 else
5647                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5648                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5649                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5650         }
5651 }
5652
5653 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5654 {
5655         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5656         struct drm_device *dev = crtc->base.dev;
5657         struct drm_i915_private *dev_priv = to_i915(dev);
5658
5659         if (!crtc_state->ips_enabled)
5660                 return;
5661
5662         /*
5663          * We can only enable IPS after we enable a plane and wait for a vblank
5664          * This function is called from post_plane_update, which is run after
5665          * a vblank wait.
5666          */
5667         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5668
5669         if (IS_BROADWELL(dev_priv)) {
5670                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5671                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5672                 /* Quoting Art Runyan: "its not safe to expect any particular
5673                  * value in IPS_CTL bit 31 after enabling IPS through the
5674                  * mailbox." Moreover, the mailbox may return a bogus state,
5675                  * so we need to just enable it and continue on.
5676                  */
5677         } else {
5678                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5679                 /* The bit only becomes 1 in the next vblank, so this wait here
5680                  * is essentially intel_wait_for_vblank. If we don't have this
5681                  * and don't wait for vblanks until the end of crtc_enable, then
5682                  * the HW state readout code will complain that the expected
5683                  * IPS_CTL value is not the one we read. */
5684                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
5685                         DRM_ERROR("Timed out waiting for IPS enable\n");
5686         }
5687 }
5688
5689 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5690 {
5691         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5692         struct drm_device *dev = crtc->base.dev;
5693         struct drm_i915_private *dev_priv = to_i915(dev);
5694
5695         if (!crtc_state->ips_enabled)
5696                 return;
5697
5698         if (IS_BROADWELL(dev_priv)) {
5699                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5700                 /*
5701                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5702                  * 42ms timeout value leads to occasional timeouts so use 100ms
5703                  * instead.
5704                  */
5705                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
5706                         DRM_ERROR("Timed out waiting for IPS disable\n");
5707         } else {
5708                 I915_WRITE(IPS_CTL, 0);
5709                 POSTING_READ(IPS_CTL);
5710         }
5711
5712         /* We need to wait for a vblank before we can disable the plane. */
5713         intel_wait_for_vblank(dev_priv, crtc->pipe);
5714 }
5715
5716 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5717 {
5718         if (intel_crtc->overlay) {
5719                 struct drm_device *dev = intel_crtc->base.dev;
5720
5721                 mutex_lock(&dev->struct_mutex);
5722                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5723                 mutex_unlock(&dev->struct_mutex);
5724         }
5725
5726         /* Let userspace switch the overlay on again. In most cases userspace
5727          * has to recompute where to put it anyway.
5728          */
5729 }
5730
5731 /**
5732  * intel_post_enable_primary - Perform operations after enabling primary plane
5733  * @crtc: the CRTC whose primary plane was just enabled
5734  * @new_crtc_state: the enabling state
5735  *
5736  * Performs potentially sleeping operations that must be done after the primary
5737  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5738  * called due to an explicit primary plane update, or due to an implicit
5739  * re-enable that is caused when a sprite plane is updated to no longer
5740  * completely hide the primary plane.
5741  */
5742 static void
5743 intel_post_enable_primary(struct drm_crtc *crtc,
5744                           const struct intel_crtc_state *new_crtc_state)
5745 {
5746         struct drm_device *dev = crtc->dev;
5747         struct drm_i915_private *dev_priv = to_i915(dev);
5748         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5749         enum pipe pipe = intel_crtc->pipe;
5750
5751         /*
5752          * Gen2 reports pipe underruns whenever all planes are disabled.
5753          * So don't enable underrun reporting before at least some planes
5754          * are enabled.
5755          * FIXME: Need to fix the logic to work when we turn off all planes
5756          * but leave the pipe running.
5757          */
5758         if (IS_GEN(dev_priv, 2))
5759                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5760
5761         /* Underruns don't always raise interrupts, so check manually. */
5762         intel_check_cpu_fifo_underruns(dev_priv);
5763         intel_check_pch_fifo_underruns(dev_priv);
5764 }
5765
5766 /* FIXME get rid of this and use pre_plane_update */
5767 static void
5768 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5769 {
5770         struct drm_device *dev = crtc->dev;
5771         struct drm_i915_private *dev_priv = to_i915(dev);
5772         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5773         enum pipe pipe = intel_crtc->pipe;
5774
5775         /*
5776          * Gen2 reports pipe underruns whenever all planes are disabled.
5777          * So disable underrun reporting before all the planes get disabled.
5778          */
5779         if (IS_GEN(dev_priv, 2))
5780                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5781
5782         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5783
5784         /*
5785          * Vblank time updates from the shadow to live plane control register
5786          * are blocked if the memory self-refresh mode is active at that
5787          * moment. So to make sure the plane gets truly disabled, disable
5788          * first the self-refresh mode. The self-refresh enable bit in turn
5789          * will be checked/applied by the HW only at the next frame start
5790          * event which is after the vblank start event, so we need to have a
5791          * wait-for-vblank between disabling the plane and the pipe.
5792          */
5793         if (HAS_GMCH(dev_priv) &&
5794             intel_set_memory_cxsr(dev_priv, false))
5795                 intel_wait_for_vblank(dev_priv, pipe);
5796 }
5797
5798 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5799                                        const struct intel_crtc_state *new_crtc_state)
5800 {
5801         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5802         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5803
5804         if (!old_crtc_state->ips_enabled)
5805                 return false;
5806
5807         if (needs_modeset(new_crtc_state))
5808                 return true;
5809
5810         /*
5811          * Workaround : Do not read or write the pipe palette/gamma data while
5812          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5813          *
5814          * Disable IPS before we program the LUT.
5815          */
5816         if (IS_HASWELL(dev_priv) &&
5817             (new_crtc_state->base.color_mgmt_changed ||
5818              new_crtc_state->update_pipe) &&
5819             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5820                 return true;
5821
5822         return !new_crtc_state->ips_enabled;
5823 }
5824
5825 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5826                                        const struct intel_crtc_state *new_crtc_state)
5827 {
5828         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5829         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5830
5831         if (!new_crtc_state->ips_enabled)
5832                 return false;
5833
5834         if (needs_modeset(new_crtc_state))
5835                 return true;
5836
5837         /*
5838          * Workaround : Do not read or write the pipe palette/gamma data while
5839          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5840          *
5841          * Re-enable IPS after the LUT has been programmed.
5842          */
5843         if (IS_HASWELL(dev_priv) &&
5844             (new_crtc_state->base.color_mgmt_changed ||
5845              new_crtc_state->update_pipe) &&
5846             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5847                 return true;
5848
5849         /*
5850          * We can't read out IPS on broadwell, assume the worst and
5851          * forcibly enable IPS on the first fastset.
5852          */
5853         if (new_crtc_state->update_pipe &&
5854             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5855                 return true;
5856
5857         return !old_crtc_state->ips_enabled;
5858 }
5859
5860 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5861                           const struct intel_crtc_state *crtc_state)
5862 {
5863         if (!crtc_state->nv12_planes)
5864                 return false;
5865
5866         /* WA Display #0827: Gen9:all */
5867         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
5868                 return true;
5869
5870         return false;
5871 }
5872
5873 static bool needs_scalerclk_wa(struct drm_i915_private *dev_priv,
5874                                const struct intel_crtc_state *crtc_state)
5875 {
5876         /* Wa_2006604312:icl */
5877         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
5878                 return true;
5879
5880         return false;
5881 }
5882
5883 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5884 {
5885         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5886         struct drm_device *dev = crtc->base.dev;
5887         struct drm_i915_private *dev_priv = to_i915(dev);
5888         struct drm_atomic_state *state = old_crtc_state->base.state;
5889         struct intel_crtc_state *pipe_config =
5890                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(state),
5891                                                 crtc);
5892         struct drm_plane *primary = crtc->base.primary;
5893         struct drm_plane_state *old_primary_state =
5894                 drm_atomic_get_old_plane_state(state, primary);
5895
5896         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5897
5898         if (pipe_config->update_wm_post && pipe_config->base.active)
5899                 intel_update_watermarks(crtc);
5900
5901         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5902                 hsw_enable_ips(pipe_config);
5903
5904         if (old_primary_state) {
5905                 struct drm_plane_state *new_primary_state =
5906                         drm_atomic_get_new_plane_state(state, primary);
5907
5908                 intel_fbc_post_update(crtc);
5909
5910                 if (new_primary_state->visible &&
5911                     (needs_modeset(pipe_config) ||
5912                      !old_primary_state->visible))
5913                         intel_post_enable_primary(&crtc->base, pipe_config);
5914         }
5915
5916         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5917             !needs_nv12_wa(dev_priv, pipe_config))
5918                 skl_wa_827(dev_priv, crtc->pipe, false);
5919
5920         if (needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5921             !needs_scalerclk_wa(dev_priv, pipe_config))
5922                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, false);
5923 }
5924
5925 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5926                                    struct intel_crtc_state *pipe_config)
5927 {
5928         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5929         struct drm_device *dev = crtc->base.dev;
5930         struct drm_i915_private *dev_priv = to_i915(dev);
5931         struct drm_atomic_state *state = old_crtc_state->base.state;
5932         struct drm_plane *primary = crtc->base.primary;
5933         struct drm_plane_state *old_primary_state =
5934                 drm_atomic_get_old_plane_state(state, primary);
5935         bool modeset = needs_modeset(pipe_config);
5936         struct intel_atomic_state *intel_state =
5937                 to_intel_atomic_state(state);
5938
5939         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5940                 hsw_disable_ips(old_crtc_state);
5941
5942         if (old_primary_state) {
5943                 struct intel_plane_state *new_primary_state =
5944                         intel_atomic_get_new_plane_state(intel_state,
5945                                                          to_intel_plane(primary));
5946
5947                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5948                 /*
5949                  * Gen2 reports pipe underruns whenever all planes are disabled.
5950                  * So disable underrun reporting before all the planes get disabled.
5951                  */
5952                 if (IS_GEN(dev_priv, 2) && old_primary_state->visible &&
5953                     (modeset || !new_primary_state->base.visible))
5954                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5955         }
5956
5957         /* Display WA 827 */
5958         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5959             needs_nv12_wa(dev_priv, pipe_config))
5960                 skl_wa_827(dev_priv, crtc->pipe, true);
5961
5962         /* Wa_2006604312:icl */
5963         if (!needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5964             needs_scalerclk_wa(dev_priv, pipe_config))
5965                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, true);
5966
5967         /*
5968          * Vblank time updates from the shadow to live plane control register
5969          * are blocked if the memory self-refresh mode is active at that
5970          * moment. So to make sure the plane gets truly disabled, disable
5971          * first the self-refresh mode. The self-refresh enable bit in turn
5972          * will be checked/applied by the HW only at the next frame start
5973          * event which is after the vblank start event, so we need to have a
5974          * wait-for-vblank between disabling the plane and the pipe.
5975          */
5976         if (HAS_GMCH(dev_priv) && old_crtc_state->base.active &&
5977             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5978                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5979
5980         /*
5981          * IVB workaround: must disable low power watermarks for at least
5982          * one frame before enabling scaling.  LP watermarks can be re-enabled
5983          * when scaling is disabled.
5984          *
5985          * WaCxSRDisabledForSpriteScaling:ivb
5986          */
5987         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
5988             old_crtc_state->base.active)
5989                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5990
5991         /*
5992          * If we're doing a modeset, we're done.  No need to do any pre-vblank
5993          * watermark programming here.
5994          */
5995         if (needs_modeset(pipe_config))
5996                 return;
5997
5998         /*
5999          * For platforms that support atomic watermarks, program the
6000          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6001          * will be the intermediate values that are safe for both pre- and
6002          * post- vblank; when vblank happens, the 'active' values will be set
6003          * to the final 'target' values and we'll do this again to get the
6004          * optimal watermarks.  For gen9+ platforms, the values we program here
6005          * will be the final target values which will get automatically latched
6006          * at vblank time; no further programming will be necessary.
6007          *
6008          * If a platform hasn't been transitioned to atomic watermarks yet,
6009          * we'll continue to update watermarks the old way, if flags tell
6010          * us to.
6011          */
6012         if (dev_priv->display.initial_watermarks != NULL)
6013                 dev_priv->display.initial_watermarks(intel_state,
6014                                                      pipe_config);
6015         else if (pipe_config->update_wm_pre)
6016                 intel_update_watermarks(crtc);
6017 }
6018
6019 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6020                                       struct intel_crtc *crtc)
6021 {
6022         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6023         const struct intel_crtc_state *new_crtc_state =
6024                 intel_atomic_get_new_crtc_state(state, crtc);
6025         unsigned int update_mask = new_crtc_state->update_planes;
6026         const struct intel_plane_state *old_plane_state;
6027         struct intel_plane *plane;
6028         unsigned fb_bits = 0;
6029         int i;
6030
6031         intel_crtc_dpms_overlay_disable(crtc);
6032
6033         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6034                 if (crtc->pipe != plane->pipe ||
6035                     !(update_mask & BIT(plane->id)))
6036                         continue;
6037
6038                 intel_disable_plane(plane, new_crtc_state);
6039
6040                 if (old_plane_state->base.visible)
6041                         fb_bits |= plane->frontbuffer_bit;
6042         }
6043
6044         intel_frontbuffer_flip(dev_priv, fb_bits);
6045 }
6046
6047 /*
6048  * intel_connector_primary_encoder - get the primary encoder for a connector
6049  * @connector: connector for which to return the encoder
6050  *
6051  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6052  * all connectors to their encoder, except for DP-MST connectors which have
6053  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6054  * pointed to by as many DP-MST connectors as there are pipes.
6055  */
6056 static struct intel_encoder *
6057 intel_connector_primary_encoder(struct intel_connector *connector)
6058 {
6059         struct intel_encoder *encoder;
6060
6061         if (connector->mst_port)
6062                 return &dp_to_dig_port(connector->mst_port)->base;
6063
6064         encoder = intel_attached_encoder(&connector->base);
6065         WARN_ON(!encoder);
6066
6067         return encoder;
6068 }
6069
6070 static bool
6071 intel_connector_needs_modeset(struct intel_atomic_state *state,
6072                               const struct drm_connector_state *old_conn_state,
6073                               const struct drm_connector_state *new_conn_state)
6074 {
6075         struct intel_crtc *old_crtc = old_conn_state->crtc ?
6076                                       to_intel_crtc(old_conn_state->crtc) : NULL;
6077         struct intel_crtc *new_crtc = new_conn_state->crtc ?
6078                                       to_intel_crtc(new_conn_state->crtc) : NULL;
6079
6080         return new_crtc != old_crtc ||
6081                (new_crtc &&
6082                 needs_modeset(intel_atomic_get_new_crtc_state(state, new_crtc)));
6083 }
6084
6085 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6086 {
6087         struct drm_connector_state *old_conn_state;
6088         struct drm_connector_state *new_conn_state;
6089         struct drm_connector *conn;
6090         int i;
6091
6092         for_each_oldnew_connector_in_state(&state->base, conn,
6093                                            old_conn_state, new_conn_state, i) {
6094                 struct intel_encoder *encoder;
6095                 struct intel_crtc *crtc;
6096
6097                 if (!intel_connector_needs_modeset(state,
6098                                                    old_conn_state,
6099                                                    new_conn_state))
6100                         continue;
6101
6102                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6103                 if (!encoder->update_prepare)
6104                         continue;
6105
6106                 crtc = new_conn_state->crtc ?
6107                         to_intel_crtc(new_conn_state->crtc) : NULL;
6108                 encoder->update_prepare(state, encoder, crtc);
6109         }
6110 }
6111
6112 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6113 {
6114         struct drm_connector_state *old_conn_state;
6115         struct drm_connector_state *new_conn_state;
6116         struct drm_connector *conn;
6117         int i;
6118
6119         for_each_oldnew_connector_in_state(&state->base, conn,
6120                                            old_conn_state, new_conn_state, i) {
6121                 struct intel_encoder *encoder;
6122                 struct intel_crtc *crtc;
6123
6124                 if (!intel_connector_needs_modeset(state,
6125                                                    old_conn_state,
6126                                                    new_conn_state))
6127                         continue;
6128
6129                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6130                 if (!encoder->update_complete)
6131                         continue;
6132
6133                 crtc = new_conn_state->crtc ?
6134                         to_intel_crtc(new_conn_state->crtc) : NULL;
6135                 encoder->update_complete(state, encoder, crtc);
6136         }
6137 }
6138
6139 static void intel_encoders_pre_pll_enable(struct intel_crtc *crtc,
6140                                           struct intel_crtc_state *crtc_state,
6141                                           struct intel_atomic_state *state)
6142 {
6143         struct drm_connector_state *conn_state;
6144         struct drm_connector *conn;
6145         int i;
6146
6147         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6148                 struct intel_encoder *encoder =
6149                         to_intel_encoder(conn_state->best_encoder);
6150
6151                 if (conn_state->crtc != &crtc->base)
6152                         continue;
6153
6154                 if (encoder->pre_pll_enable)
6155                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6156         }
6157 }
6158
6159 static void intel_encoders_pre_enable(struct intel_crtc *crtc,
6160                                       struct intel_crtc_state *crtc_state,
6161                                       struct intel_atomic_state *state)
6162 {
6163         struct drm_connector_state *conn_state;
6164         struct drm_connector *conn;
6165         int i;
6166
6167         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6168                 struct intel_encoder *encoder =
6169                         to_intel_encoder(conn_state->best_encoder);
6170
6171                 if (conn_state->crtc != &crtc->base)
6172                         continue;
6173
6174                 if (encoder->pre_enable)
6175                         encoder->pre_enable(encoder, crtc_state, conn_state);
6176         }
6177 }
6178
6179 static void intel_encoders_enable(struct intel_crtc *crtc,
6180                                   struct intel_crtc_state *crtc_state,
6181                                   struct intel_atomic_state *state)
6182 {
6183         struct drm_connector_state *conn_state;
6184         struct drm_connector *conn;
6185         int i;
6186
6187         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6188                 struct intel_encoder *encoder =
6189                         to_intel_encoder(conn_state->best_encoder);
6190
6191                 if (conn_state->crtc != &crtc->base)
6192                         continue;
6193
6194                 if (encoder->enable)
6195                         encoder->enable(encoder, crtc_state, conn_state);
6196                 intel_opregion_notify_encoder(encoder, true);
6197         }
6198 }
6199
6200 static void intel_encoders_disable(struct intel_crtc *crtc,
6201                                    struct intel_crtc_state *old_crtc_state,
6202                                    struct intel_atomic_state *state)
6203 {
6204         struct drm_connector_state *old_conn_state;
6205         struct drm_connector *conn;
6206         int i;
6207
6208         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6209                 struct intel_encoder *encoder =
6210                         to_intel_encoder(old_conn_state->best_encoder);
6211
6212                 if (old_conn_state->crtc != &crtc->base)
6213                         continue;
6214
6215                 intel_opregion_notify_encoder(encoder, false);
6216                 if (encoder->disable)
6217                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6218         }
6219 }
6220
6221 static void intel_encoders_post_disable(struct intel_crtc *crtc,
6222                                         struct intel_crtc_state *old_crtc_state,
6223                                         struct intel_atomic_state *state)
6224 {
6225         struct drm_connector_state *old_conn_state;
6226         struct drm_connector *conn;
6227         int i;
6228
6229         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6230                 struct intel_encoder *encoder =
6231                         to_intel_encoder(old_conn_state->best_encoder);
6232
6233                 if (old_conn_state->crtc != &crtc->base)
6234                         continue;
6235
6236                 if (encoder->post_disable)
6237                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6238         }
6239 }
6240
6241 static void intel_encoders_post_pll_disable(struct intel_crtc *crtc,
6242                                             struct intel_crtc_state *old_crtc_state,
6243                                             struct intel_atomic_state *state)
6244 {
6245         struct drm_connector_state *old_conn_state;
6246         struct drm_connector *conn;
6247         int i;
6248
6249         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6250                 struct intel_encoder *encoder =
6251                         to_intel_encoder(old_conn_state->best_encoder);
6252
6253                 if (old_conn_state->crtc != &crtc->base)
6254                         continue;
6255
6256                 if (encoder->post_pll_disable)
6257                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6258         }
6259 }
6260
6261 static void intel_encoders_update_pipe(struct intel_crtc *crtc,
6262                                        struct intel_crtc_state *crtc_state,
6263                                        struct intel_atomic_state *state)
6264 {
6265         struct drm_connector_state *conn_state;
6266         struct drm_connector *conn;
6267         int i;
6268
6269         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6270                 struct intel_encoder *encoder =
6271                         to_intel_encoder(conn_state->best_encoder);
6272
6273                 if (conn_state->crtc != &crtc->base)
6274                         continue;
6275
6276                 if (encoder->update_pipe)
6277                         encoder->update_pipe(encoder, crtc_state, conn_state);
6278         }
6279 }
6280
6281 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6282 {
6283         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6284         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6285
6286         plane->disable_plane(plane, crtc_state);
6287 }
6288
6289 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
6290                                  struct intel_atomic_state *state)
6291 {
6292         struct drm_crtc *crtc = pipe_config->base.crtc;
6293         struct drm_device *dev = crtc->dev;
6294         struct drm_i915_private *dev_priv = to_i915(dev);
6295         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6296         enum pipe pipe = intel_crtc->pipe;
6297
6298         if (WARN_ON(intel_crtc->active))
6299                 return;
6300
6301         /*
6302          * Sometimes spurious CPU pipe underruns happen during FDI
6303          * training, at least with VGA+HDMI cloning. Suppress them.
6304          *
6305          * On ILK we get an occasional spurious CPU pipe underruns
6306          * between eDP port A enable and vdd enable. Also PCH port
6307          * enable seems to result in the occasional CPU pipe underrun.
6308          *
6309          * Spurious PCH underruns also occur during PCH enabling.
6310          */
6311         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6312         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6313
6314         if (pipe_config->has_pch_encoder)
6315                 intel_prepare_shared_dpll(pipe_config);
6316
6317         if (intel_crtc_has_dp_encoder(pipe_config))
6318                 intel_dp_set_m_n(pipe_config, M1_N1);
6319
6320         intel_set_pipe_timings(pipe_config);
6321         intel_set_pipe_src_size(pipe_config);
6322
6323         if (pipe_config->has_pch_encoder) {
6324                 intel_cpu_transcoder_set_m_n(pipe_config,
6325                                              &pipe_config->fdi_m_n, NULL);
6326         }
6327
6328         ironlake_set_pipeconf(pipe_config);
6329
6330         intel_crtc->active = true;
6331
6332         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6333
6334         if (pipe_config->has_pch_encoder) {
6335                 /* Note: FDI PLL enabling _must_ be done before we enable the
6336                  * cpu pipes, hence this is separate from all the other fdi/pch
6337                  * enabling. */
6338                 ironlake_fdi_pll_enable(pipe_config);
6339         } else {
6340                 assert_fdi_tx_disabled(dev_priv, pipe);
6341                 assert_fdi_rx_disabled(dev_priv, pipe);
6342         }
6343
6344         ironlake_pfit_enable(pipe_config);
6345
6346         /*
6347          * On ILK+ LUT must be loaded before the pipe is running but with
6348          * clocks enabled
6349          */
6350         intel_color_load_luts(pipe_config);
6351         intel_color_commit(pipe_config);
6352         /* update DSPCNTR to configure gamma for pipe bottom color */
6353         intel_disable_primary_plane(pipe_config);
6354
6355         if (dev_priv->display.initial_watermarks != NULL)
6356                 dev_priv->display.initial_watermarks(state, pipe_config);
6357         intel_enable_pipe(pipe_config);
6358
6359         if (pipe_config->has_pch_encoder)
6360                 ironlake_pch_enable(state, pipe_config);
6361
6362         assert_vblank_disabled(crtc);
6363         intel_crtc_vblank_on(pipe_config);
6364
6365         intel_encoders_enable(intel_crtc, pipe_config, state);
6366
6367         if (HAS_PCH_CPT(dev_priv))
6368                 cpt_verify_modeset(dev, intel_crtc->pipe);
6369
6370         /*
6371          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6372          * And a second vblank wait is needed at least on ILK with
6373          * some interlaced HDMI modes. Let's do the double wait always
6374          * in case there are more corner cases we don't know about.
6375          */
6376         if (pipe_config->has_pch_encoder) {
6377                 intel_wait_for_vblank(dev_priv, pipe);
6378                 intel_wait_for_vblank(dev_priv, pipe);
6379         }
6380         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6381         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6382 }
6383
6384 /* IPS only exists on ULT machines and is tied to pipe A. */
6385 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6386 {
6387         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6388 }
6389
6390 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6391                                             enum pipe pipe, bool apply)
6392 {
6393         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6394         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6395
6396         if (apply)
6397                 val |= mask;
6398         else
6399                 val &= ~mask;
6400
6401         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6402 }
6403
6404 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6405 {
6406         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6407         enum pipe pipe = crtc->pipe;
6408         u32 val;
6409
6410         val = MBUS_DBOX_A_CREDIT(2);
6411
6412         if (INTEL_GEN(dev_priv) >= 12) {
6413                 val |= MBUS_DBOX_BW_CREDIT(2);
6414                 val |= MBUS_DBOX_B_CREDIT(12);
6415         } else {
6416                 val |= MBUS_DBOX_BW_CREDIT(1);
6417                 val |= MBUS_DBOX_B_CREDIT(8);
6418         }
6419
6420         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6421 }
6422
6423 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
6424                                 struct intel_atomic_state *state)
6425 {
6426         struct drm_crtc *crtc = pipe_config->base.crtc;
6427         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6428         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6429         enum pipe pipe = intel_crtc->pipe, hsw_workaround_pipe;
6430         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
6431         bool psl_clkgate_wa;
6432
6433         if (WARN_ON(intel_crtc->active))
6434                 return;
6435
6436         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6437
6438         if (pipe_config->shared_dpll)
6439                 intel_enable_shared_dpll(pipe_config);
6440
6441         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6442
6443         if (intel_crtc_has_dp_encoder(pipe_config))
6444                 intel_dp_set_m_n(pipe_config, M1_N1);
6445
6446         if (!transcoder_is_dsi(cpu_transcoder))
6447                 intel_set_pipe_timings(pipe_config);
6448
6449         intel_set_pipe_src_size(pipe_config);
6450
6451         if (cpu_transcoder != TRANSCODER_EDP &&
6452             !transcoder_is_dsi(cpu_transcoder)) {
6453                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6454                            pipe_config->pixel_multiplier - 1);
6455         }
6456
6457         if (pipe_config->has_pch_encoder) {
6458                 intel_cpu_transcoder_set_m_n(pipe_config,
6459                                              &pipe_config->fdi_m_n, NULL);
6460         }
6461
6462         if (!transcoder_is_dsi(cpu_transcoder))
6463                 haswell_set_pipeconf(pipe_config);
6464
6465         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6466                 bdw_set_pipemisc(pipe_config);
6467
6468         intel_crtc->active = true;
6469
6470         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6471         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6472                          pipe_config->pch_pfit.enabled;
6473         if (psl_clkgate_wa)
6474                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6475
6476         if (INTEL_GEN(dev_priv) >= 9)
6477                 skylake_pfit_enable(pipe_config);
6478         else
6479                 ironlake_pfit_enable(pipe_config);
6480
6481         /*
6482          * On ILK+ LUT must be loaded before the pipe is running but with
6483          * clocks enabled
6484          */
6485         intel_color_load_luts(pipe_config);
6486         intel_color_commit(pipe_config);
6487         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6488         if (INTEL_GEN(dev_priv) < 9)
6489                 intel_disable_primary_plane(pipe_config);
6490
6491         if (INTEL_GEN(dev_priv) >= 11)
6492                 icl_set_pipe_chicken(intel_crtc);
6493
6494         intel_ddi_set_pipe_settings(pipe_config);
6495         if (!transcoder_is_dsi(cpu_transcoder))
6496                 intel_ddi_enable_transcoder_func(pipe_config);
6497
6498         if (dev_priv->display.initial_watermarks != NULL)
6499                 dev_priv->display.initial_watermarks(state, pipe_config);
6500
6501         if (INTEL_GEN(dev_priv) >= 11)
6502                 icl_pipe_mbus_enable(intel_crtc);
6503
6504         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6505         if (!transcoder_is_dsi(cpu_transcoder))
6506                 intel_enable_pipe(pipe_config);
6507
6508         if (pipe_config->has_pch_encoder)
6509                 lpt_pch_enable(state, pipe_config);
6510
6511         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
6512                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
6513
6514         assert_vblank_disabled(crtc);
6515         intel_crtc_vblank_on(pipe_config);
6516
6517         intel_encoders_enable(intel_crtc, pipe_config, state);
6518
6519         if (psl_clkgate_wa) {
6520                 intel_wait_for_vblank(dev_priv, pipe);
6521                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6522         }
6523
6524         /* If we change the relative order between pipe/planes enabling, we need
6525          * to change the workaround. */
6526         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
6527         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6528                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6529                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6530         }
6531 }
6532
6533 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6534 {
6535         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6536         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6537         enum pipe pipe = crtc->pipe;
6538
6539         /* To avoid upsetting the power well on haswell only disable the pfit if
6540          * it's in use. The hw state code will make sure we get this right. */
6541         if (old_crtc_state->pch_pfit.enabled) {
6542                 I915_WRITE(PF_CTL(pipe), 0);
6543                 I915_WRITE(PF_WIN_POS(pipe), 0);
6544                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6545         }
6546 }
6547
6548 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
6549                                   struct intel_atomic_state *state)
6550 {
6551         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6552         struct drm_device *dev = crtc->dev;
6553         struct drm_i915_private *dev_priv = to_i915(dev);
6554         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6555         enum pipe pipe = intel_crtc->pipe;
6556
6557         /*
6558          * Sometimes spurious CPU pipe underruns happen when the
6559          * pipe is already disabled, but FDI RX/TX is still enabled.
6560          * Happens at least with VGA+HDMI cloning. Suppress them.
6561          */
6562         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6563         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6564
6565         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6566
6567         drm_crtc_vblank_off(crtc);
6568         assert_vblank_disabled(crtc);
6569
6570         intel_disable_pipe(old_crtc_state);
6571
6572         ironlake_pfit_disable(old_crtc_state);
6573
6574         if (old_crtc_state->has_pch_encoder)
6575                 ironlake_fdi_disable(crtc);
6576
6577         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6578
6579         if (old_crtc_state->has_pch_encoder) {
6580                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6581
6582                 if (HAS_PCH_CPT(dev_priv)) {
6583                         i915_reg_t reg;
6584                         u32 temp;
6585
6586                         /* disable TRANS_DP_CTL */
6587                         reg = TRANS_DP_CTL(pipe);
6588                         temp = I915_READ(reg);
6589                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6590                                   TRANS_DP_PORT_SEL_MASK);
6591                         temp |= TRANS_DP_PORT_SEL_NONE;
6592                         I915_WRITE(reg, temp);
6593
6594                         /* disable DPLL_SEL */
6595                         temp = I915_READ(PCH_DPLL_SEL);
6596                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6597                         I915_WRITE(PCH_DPLL_SEL, temp);
6598                 }
6599
6600                 ironlake_fdi_pll_disable(intel_crtc);
6601         }
6602
6603         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6604         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6605 }
6606
6607 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
6608                                  struct intel_atomic_state *state)
6609 {
6610         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6611         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6612         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6613         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
6614
6615         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6616
6617         drm_crtc_vblank_off(crtc);
6618         assert_vblank_disabled(crtc);
6619
6620         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6621         if (!transcoder_is_dsi(cpu_transcoder))
6622                 intel_disable_pipe(old_crtc_state);
6623
6624         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
6625                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
6626
6627         if (!transcoder_is_dsi(cpu_transcoder))
6628                 intel_ddi_disable_transcoder_func(old_crtc_state);
6629
6630         intel_dsc_disable(old_crtc_state);
6631
6632         if (INTEL_GEN(dev_priv) >= 9)
6633                 skylake_scaler_disable(intel_crtc);
6634         else
6635                 ironlake_pfit_disable(old_crtc_state);
6636
6637         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6638
6639         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
6640 }
6641
6642 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6643 {
6644         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6645         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6646
6647         if (!crtc_state->gmch_pfit.control)
6648                 return;
6649
6650         /*
6651          * The panel fitter should only be adjusted whilst the pipe is disabled,
6652          * according to register description and PRM.
6653          */
6654         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6655         assert_pipe_disabled(dev_priv, crtc->pipe);
6656
6657         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6658         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6659
6660         /* Border color in case we don't scale up to the full screen. Black by
6661          * default, change to something else for debugging. */
6662         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6663 }
6664
6665 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6666 {
6667         if (phy == PHY_NONE)
6668                 return false;
6669
6670         if (IS_ELKHARTLAKE(dev_priv))
6671                 return phy <= PHY_C;
6672
6673         if (INTEL_GEN(dev_priv) >= 11)
6674                 return phy <= PHY_B;
6675
6676         return false;
6677 }
6678
6679 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
6680 {
6681         if (INTEL_GEN(dev_priv) >= 12)
6682                 return phy >= PHY_D && phy <= PHY_I;
6683
6684         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
6685                 return phy >= PHY_C && phy <= PHY_F;
6686
6687         return false;
6688 }
6689
6690 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
6691 {
6692         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
6693                 return PHY_A;
6694
6695         return (enum phy)port;
6696 }
6697
6698 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
6699 {
6700         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
6701                 return PORT_TC_NONE;
6702
6703         if (INTEL_GEN(dev_priv) >= 12)
6704                 return port - PORT_D;
6705
6706         return port - PORT_C;
6707 }
6708
6709 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
6710 {
6711         switch (port) {
6712         case PORT_A:
6713                 return POWER_DOMAIN_PORT_DDI_A_LANES;
6714         case PORT_B:
6715                 return POWER_DOMAIN_PORT_DDI_B_LANES;
6716         case PORT_C:
6717                 return POWER_DOMAIN_PORT_DDI_C_LANES;
6718         case PORT_D:
6719                 return POWER_DOMAIN_PORT_DDI_D_LANES;
6720         case PORT_E:
6721                 return POWER_DOMAIN_PORT_DDI_E_LANES;
6722         case PORT_F:
6723                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6724         default:
6725                 MISSING_CASE(port);
6726                 return POWER_DOMAIN_PORT_OTHER;
6727         }
6728 }
6729
6730 enum intel_display_power_domain
6731 intel_aux_power_domain(struct intel_digital_port *dig_port)
6732 {
6733         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
6734         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
6735
6736         if (intel_phy_is_tc(dev_priv, phy) &&
6737             dig_port->tc_mode == TC_PORT_TBT_ALT) {
6738                 switch (dig_port->aux_ch) {
6739                 case AUX_CH_C:
6740                         return POWER_DOMAIN_AUX_C_TBT;
6741                 case AUX_CH_D:
6742                         return POWER_DOMAIN_AUX_D_TBT;
6743                 case AUX_CH_E:
6744                         return POWER_DOMAIN_AUX_E_TBT;
6745                 case AUX_CH_F:
6746                         return POWER_DOMAIN_AUX_F_TBT;
6747                 default:
6748                         MISSING_CASE(dig_port->aux_ch);
6749                         return POWER_DOMAIN_AUX_C_TBT;
6750                 }
6751         }
6752
6753         switch (dig_port->aux_ch) {
6754         case AUX_CH_A:
6755                 return POWER_DOMAIN_AUX_A;
6756         case AUX_CH_B:
6757                 return POWER_DOMAIN_AUX_B;
6758         case AUX_CH_C:
6759                 return POWER_DOMAIN_AUX_C;
6760         case AUX_CH_D:
6761                 return POWER_DOMAIN_AUX_D;
6762         case AUX_CH_E:
6763                 return POWER_DOMAIN_AUX_E;
6764         case AUX_CH_F:
6765                 return POWER_DOMAIN_AUX_F;
6766         default:
6767                 MISSING_CASE(dig_port->aux_ch);
6768                 return POWER_DOMAIN_AUX_A;
6769         }
6770 }
6771
6772 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6773 {
6774         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6775         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6776         struct drm_encoder *encoder;
6777         enum pipe pipe = crtc->pipe;
6778         u64 mask;
6779         enum transcoder transcoder = crtc_state->cpu_transcoder;
6780
6781         if (!crtc_state->base.active)
6782                 return 0;
6783
6784         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6785         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6786         if (crtc_state->pch_pfit.enabled ||
6787             crtc_state->pch_pfit.force_thru)
6788                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6789
6790         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
6791                                   crtc_state->base.encoder_mask) {
6792                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6793
6794                 mask |= BIT_ULL(intel_encoder->power_domain);
6795         }
6796
6797         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6798                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6799
6800         if (crtc_state->shared_dpll)
6801                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
6802
6803         return mask;
6804 }
6805
6806 static u64
6807 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6808 {
6809         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6810         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6811         enum intel_display_power_domain domain;
6812         u64 domains, new_domains, old_domains;
6813
6814         old_domains = crtc->enabled_power_domains;
6815         crtc->enabled_power_domains = new_domains =
6816                 get_crtc_power_domains(crtc_state);
6817
6818         domains = new_domains & ~old_domains;
6819
6820         for_each_power_domain(domain, domains)
6821                 intel_display_power_get(dev_priv, domain);
6822
6823         return old_domains & ~new_domains;
6824 }
6825
6826 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6827                                       u64 domains)
6828 {
6829         enum intel_display_power_domain domain;
6830
6831         for_each_power_domain(domain, domains)
6832                 intel_display_power_put_unchecked(dev_priv, domain);
6833 }
6834
6835 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6836                                    struct intel_atomic_state *state)
6837 {
6838         struct drm_crtc *crtc = pipe_config->base.crtc;
6839         struct drm_device *dev = crtc->dev;
6840         struct drm_i915_private *dev_priv = to_i915(dev);
6841         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6842         enum pipe pipe = intel_crtc->pipe;
6843
6844         if (WARN_ON(intel_crtc->active))
6845                 return;
6846
6847         if (intel_crtc_has_dp_encoder(pipe_config))
6848                 intel_dp_set_m_n(pipe_config, M1_N1);
6849
6850         intel_set_pipe_timings(pipe_config);
6851         intel_set_pipe_src_size(pipe_config);
6852
6853         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6854                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6855                 I915_WRITE(CHV_CANVAS(pipe), 0);
6856         }
6857
6858         i9xx_set_pipeconf(pipe_config);
6859
6860         intel_crtc->active = true;
6861
6862         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6863
6864         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6865
6866         if (IS_CHERRYVIEW(dev_priv)) {
6867                 chv_prepare_pll(intel_crtc, pipe_config);
6868                 chv_enable_pll(intel_crtc, pipe_config);
6869         } else {
6870                 vlv_prepare_pll(intel_crtc, pipe_config);
6871                 vlv_enable_pll(intel_crtc, pipe_config);
6872         }
6873
6874         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6875
6876         i9xx_pfit_enable(pipe_config);
6877
6878         intel_color_load_luts(pipe_config);
6879         intel_color_commit(pipe_config);
6880         /* update DSPCNTR to configure gamma for pipe bottom color */
6881         intel_disable_primary_plane(pipe_config);
6882
6883         dev_priv->display.initial_watermarks(state, pipe_config);
6884         intel_enable_pipe(pipe_config);
6885
6886         assert_vblank_disabled(crtc);
6887         intel_crtc_vblank_on(pipe_config);
6888
6889         intel_encoders_enable(intel_crtc, pipe_config, state);
6890 }
6891
6892 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6893 {
6894         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6895         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6896
6897         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6898         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6899 }
6900
6901 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6902                              struct intel_atomic_state *state)
6903 {
6904         struct drm_crtc *crtc = pipe_config->base.crtc;
6905         struct drm_device *dev = crtc->dev;
6906         struct drm_i915_private *dev_priv = to_i915(dev);
6907         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6908         enum pipe pipe = intel_crtc->pipe;
6909
6910         if (WARN_ON(intel_crtc->active))
6911                 return;
6912
6913         i9xx_set_pll_dividers(pipe_config);
6914
6915         if (intel_crtc_has_dp_encoder(pipe_config))
6916                 intel_dp_set_m_n(pipe_config, M1_N1);
6917
6918         intel_set_pipe_timings(pipe_config);
6919         intel_set_pipe_src_size(pipe_config);
6920
6921         i9xx_set_pipeconf(pipe_config);
6922
6923         intel_crtc->active = true;
6924
6925         if (!IS_GEN(dev_priv, 2))
6926                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6927
6928         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6929
6930         i9xx_enable_pll(intel_crtc, pipe_config);
6931
6932         i9xx_pfit_enable(pipe_config);
6933
6934         intel_color_load_luts(pipe_config);
6935         intel_color_commit(pipe_config);
6936         /* update DSPCNTR to configure gamma for pipe bottom color */
6937         intel_disable_primary_plane(pipe_config);
6938
6939         if (dev_priv->display.initial_watermarks != NULL)
6940                 dev_priv->display.initial_watermarks(state,
6941                                                      pipe_config);
6942         else
6943                 intel_update_watermarks(intel_crtc);
6944         intel_enable_pipe(pipe_config);
6945
6946         assert_vblank_disabled(crtc);
6947         intel_crtc_vblank_on(pipe_config);
6948
6949         intel_encoders_enable(intel_crtc, pipe_config, state);
6950 }
6951
6952 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6953 {
6954         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6955         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6956
6957         if (!old_crtc_state->gmch_pfit.control)
6958                 return;
6959
6960         assert_pipe_disabled(dev_priv, crtc->pipe);
6961
6962         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6963                       I915_READ(PFIT_CONTROL));
6964         I915_WRITE(PFIT_CONTROL, 0);
6965 }
6966
6967 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6968                               struct intel_atomic_state *state)
6969 {
6970         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6971         struct drm_device *dev = crtc->dev;
6972         struct drm_i915_private *dev_priv = to_i915(dev);
6973         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6974         enum pipe pipe = intel_crtc->pipe;
6975
6976         /*
6977          * On gen2 planes are double buffered but the pipe isn't, so we must
6978          * wait for planes to fully turn off before disabling the pipe.
6979          */
6980         if (IS_GEN(dev_priv, 2))
6981                 intel_wait_for_vblank(dev_priv, pipe);
6982
6983         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6984
6985         drm_crtc_vblank_off(crtc);
6986         assert_vblank_disabled(crtc);
6987
6988         intel_disable_pipe(old_crtc_state);
6989
6990         i9xx_pfit_disable(old_crtc_state);
6991
6992         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6993
6994         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
6995                 if (IS_CHERRYVIEW(dev_priv))
6996                         chv_disable_pll(dev_priv, pipe);
6997                 else if (IS_VALLEYVIEW(dev_priv))
6998                         vlv_disable_pll(dev_priv, pipe);
6999                 else
7000                         i9xx_disable_pll(old_crtc_state);
7001         }
7002
7003         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
7004
7005         if (!IS_GEN(dev_priv, 2))
7006                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7007
7008         if (!dev_priv->display.initial_watermarks)
7009                 intel_update_watermarks(intel_crtc);
7010
7011         /* clock the pipe down to 640x480@60 to potentially save power */
7012         if (IS_I830(dev_priv))
7013                 i830_enable_pipe(dev_priv, pipe);
7014 }
7015
7016 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
7017                                         struct drm_modeset_acquire_ctx *ctx)
7018 {
7019         struct intel_encoder *encoder;
7020         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7021         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
7022         struct intel_bw_state *bw_state =
7023                 to_intel_bw_state(dev_priv->bw_obj.state);
7024         enum intel_display_power_domain domain;
7025         struct intel_plane *plane;
7026         u64 domains;
7027         struct drm_atomic_state *state;
7028         struct intel_crtc_state *crtc_state;
7029         int ret;
7030
7031         if (!intel_crtc->active)
7032                 return;
7033
7034         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
7035                 const struct intel_plane_state *plane_state =
7036                         to_intel_plane_state(plane->base.state);
7037
7038                 if (plane_state->base.visible)
7039                         intel_plane_disable_noatomic(intel_crtc, plane);
7040         }
7041
7042         state = drm_atomic_state_alloc(crtc->dev);
7043         if (!state) {
7044                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7045                               crtc->base.id, crtc->name);
7046                 return;
7047         }
7048
7049         state->acquire_ctx = ctx;
7050
7051         /* Everything's already locked, -EDEADLK can't happen. */
7052         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
7053         ret = drm_atomic_add_affected_connectors(state, crtc);
7054
7055         WARN_ON(IS_ERR(crtc_state) || ret);
7056
7057         dev_priv->display.crtc_disable(crtc_state, to_intel_atomic_state(state));
7058
7059         drm_atomic_state_put(state);
7060
7061         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7062                       crtc->base.id, crtc->name);
7063
7064         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
7065         crtc->state->active = false;
7066         intel_crtc->active = false;
7067         crtc->enabled = false;
7068         crtc->state->connector_mask = 0;
7069         crtc->state->encoder_mask = 0;
7070
7071         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
7072                 encoder->base.crtc = NULL;
7073
7074         intel_fbc_disable(intel_crtc);
7075         intel_update_watermarks(intel_crtc);
7076         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
7077
7078         domains = intel_crtc->enabled_power_domains;
7079         for_each_power_domain(domain, domains)
7080                 intel_display_power_put_unchecked(dev_priv, domain);
7081         intel_crtc->enabled_power_domains = 0;
7082
7083         dev_priv->active_pipes &= ~BIT(intel_crtc->pipe);
7084         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
7085         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
7086
7087         bw_state->data_rate[intel_crtc->pipe] = 0;
7088         bw_state->num_active_planes[intel_crtc->pipe] = 0;
7089 }
7090
7091 /*
7092  * turn all crtc's off, but do not adjust state
7093  * This has to be paired with a call to intel_modeset_setup_hw_state.
7094  */
7095 int intel_display_suspend(struct drm_device *dev)
7096 {
7097         struct drm_i915_private *dev_priv = to_i915(dev);
7098         struct drm_atomic_state *state;
7099         int ret;
7100
7101         state = drm_atomic_helper_suspend(dev);
7102         ret = PTR_ERR_OR_ZERO(state);
7103         if (ret)
7104                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7105         else
7106                 dev_priv->modeset_restore_state = state;
7107         return ret;
7108 }
7109
7110 void intel_encoder_destroy(struct drm_encoder *encoder)
7111 {
7112         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7113
7114         drm_encoder_cleanup(encoder);
7115         kfree(intel_encoder);
7116 }
7117
7118 /* Cross check the actual hw state with our own modeset state tracking (and it's
7119  * internal consistency). */
7120 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7121                                          struct drm_connector_state *conn_state)
7122 {
7123         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7124
7125         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7126                       connector->base.base.id,
7127                       connector->base.name);
7128
7129         if (connector->get_hw_state(connector)) {
7130                 struct intel_encoder *encoder = connector->encoder;
7131
7132                 I915_STATE_WARN(!crtc_state,
7133                          "connector enabled without attached crtc\n");
7134
7135                 if (!crtc_state)
7136                         return;
7137
7138                 I915_STATE_WARN(!crtc_state->base.active,
7139                       "connector is active, but attached crtc isn't\n");
7140
7141                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7142                         return;
7143
7144                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7145                         "atomic encoder doesn't match attached encoder\n");
7146
7147                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7148                         "attached encoder crtc differs from connector crtc\n");
7149         } else {
7150                 I915_STATE_WARN(crtc_state && crtc_state->base.active,
7151                         "attached crtc is active, but connector isn't\n");
7152                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7153                         "best encoder set without crtc!\n");
7154         }
7155 }
7156
7157 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7158 {
7159         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
7160                 return crtc_state->fdi_lanes;
7161
7162         return 0;
7163 }
7164
7165 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7166                                      struct intel_crtc_state *pipe_config)
7167 {
7168         struct drm_i915_private *dev_priv = to_i915(dev);
7169         struct drm_atomic_state *state = pipe_config->base.state;
7170         struct intel_crtc *other_crtc;
7171         struct intel_crtc_state *other_crtc_state;
7172
7173         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7174                       pipe_name(pipe), pipe_config->fdi_lanes);
7175         if (pipe_config->fdi_lanes > 4) {
7176                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7177                               pipe_name(pipe), pipe_config->fdi_lanes);
7178                 return -EINVAL;
7179         }
7180
7181         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7182                 if (pipe_config->fdi_lanes > 2) {
7183                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7184                                       pipe_config->fdi_lanes);
7185                         return -EINVAL;
7186                 } else {
7187                         return 0;
7188                 }
7189         }
7190
7191         if (INTEL_INFO(dev_priv)->num_pipes == 2)
7192                 return 0;
7193
7194         /* Ivybridge 3 pipe is really complicated */
7195         switch (pipe) {
7196         case PIPE_A:
7197                 return 0;
7198         case PIPE_B:
7199                 if (pipe_config->fdi_lanes <= 2)
7200                         return 0;
7201
7202                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7203                 other_crtc_state =
7204                         intel_atomic_get_crtc_state(state, other_crtc);
7205                 if (IS_ERR(other_crtc_state))
7206                         return PTR_ERR(other_crtc_state);
7207
7208                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7209                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7210                                       pipe_name(pipe), pipe_config->fdi_lanes);
7211                         return -EINVAL;
7212                 }
7213                 return 0;
7214         case PIPE_C:
7215                 if (pipe_config->fdi_lanes > 2) {
7216                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7217                                       pipe_name(pipe), pipe_config->fdi_lanes);
7218                         return -EINVAL;
7219                 }
7220
7221                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7222                 other_crtc_state =
7223                         intel_atomic_get_crtc_state(state, other_crtc);
7224                 if (IS_ERR(other_crtc_state))
7225                         return PTR_ERR(other_crtc_state);
7226
7227                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7228                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7229                         return -EINVAL;
7230                 }
7231                 return 0;
7232         default:
7233                 BUG();
7234         }
7235 }
7236
7237 #define RETRY 1
7238 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7239                                        struct intel_crtc_state *pipe_config)
7240 {
7241         struct drm_device *dev = intel_crtc->base.dev;
7242         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7243         int lane, link_bw, fdi_dotclock, ret;
7244         bool needs_recompute = false;
7245
7246 retry:
7247         /* FDI is a binary signal running at ~2.7GHz, encoding
7248          * each output octet as 10 bits. The actual frequency
7249          * is stored as a divider into a 100MHz clock, and the
7250          * mode pixel clock is stored in units of 1KHz.
7251          * Hence the bw of each lane in terms of the mode signal
7252          * is:
7253          */
7254         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7255
7256         fdi_dotclock = adjusted_mode->crtc_clock;
7257
7258         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7259                                            pipe_config->pipe_bpp);
7260
7261         pipe_config->fdi_lanes = lane;
7262
7263         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7264                                link_bw, &pipe_config->fdi_m_n, false);
7265
7266         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7267         if (ret == -EDEADLK)
7268                 return ret;
7269
7270         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7271                 pipe_config->pipe_bpp -= 2*3;
7272                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7273                               pipe_config->pipe_bpp);
7274                 needs_recompute = true;
7275                 pipe_config->bw_constrained = true;
7276
7277                 goto retry;
7278         }
7279
7280         if (needs_recompute)
7281                 return RETRY;
7282
7283         return ret;
7284 }
7285
7286 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7287 {
7288         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7289         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7290
7291         /* IPS only exists on ULT machines and is tied to pipe A. */
7292         if (!hsw_crtc_supports_ips(crtc))
7293                 return false;
7294
7295         if (!i915_modparams.enable_ips)
7296                 return false;
7297
7298         if (crtc_state->pipe_bpp > 24)
7299                 return false;
7300
7301         /*
7302          * We compare against max which means we must take
7303          * the increased cdclk requirement into account when
7304          * calculating the new cdclk.
7305          *
7306          * Should measure whether using a lower cdclk w/o IPS
7307          */
7308         if (IS_BROADWELL(dev_priv) &&
7309             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7310                 return false;
7311
7312         return true;
7313 }
7314
7315 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7316 {
7317         struct drm_i915_private *dev_priv =
7318                 to_i915(crtc_state->base.crtc->dev);
7319         struct intel_atomic_state *intel_state =
7320                 to_intel_atomic_state(crtc_state->base.state);
7321
7322         if (!hsw_crtc_state_ips_capable(crtc_state))
7323                 return false;
7324
7325         /*
7326          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7327          * enabled and disabled dynamically based on package C states,
7328          * user space can't make reliable use of the CRCs, so let's just
7329          * completely disable it.
7330          */
7331         if (crtc_state->crc_enabled)
7332                 return false;
7333
7334         /* IPS should be fine as long as at least one plane is enabled. */
7335         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7336                 return false;
7337
7338         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7339         if (IS_BROADWELL(dev_priv) &&
7340             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7341                 return false;
7342
7343         return true;
7344 }
7345
7346 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7347 {
7348         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7349
7350         /* GDG double wide on either pipe, otherwise pipe A only */
7351         return INTEL_GEN(dev_priv) < 4 &&
7352                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7353 }
7354
7355 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7356 {
7357         u32 pixel_rate;
7358
7359         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
7360
7361         /*
7362          * We only use IF-ID interlacing. If we ever use
7363          * PF-ID we'll need to adjust the pixel_rate here.
7364          */
7365
7366         if (pipe_config->pch_pfit.enabled) {
7367                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7368                 u32 pfit_size = pipe_config->pch_pfit.size;
7369
7370                 pipe_w = pipe_config->pipe_src_w;
7371                 pipe_h = pipe_config->pipe_src_h;
7372
7373                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7374                 pfit_h = pfit_size & 0xFFFF;
7375                 if (pipe_w < pfit_w)
7376                         pipe_w = pfit_w;
7377                 if (pipe_h < pfit_h)
7378                         pipe_h = pfit_h;
7379
7380                 if (WARN_ON(!pfit_w || !pfit_h))
7381                         return pixel_rate;
7382
7383                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7384                                      pfit_w * pfit_h);
7385         }
7386
7387         return pixel_rate;
7388 }
7389
7390 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7391 {
7392         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
7393
7394         if (HAS_GMCH(dev_priv))
7395                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7396                 crtc_state->pixel_rate =
7397                         crtc_state->base.adjusted_mode.crtc_clock;
7398         else
7399                 crtc_state->pixel_rate =
7400                         ilk_pipe_pixel_rate(crtc_state);
7401 }
7402
7403 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7404                                      struct intel_crtc_state *pipe_config)
7405 {
7406         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7407         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7408         int clock_limit = dev_priv->max_dotclk_freq;
7409
7410         if (INTEL_GEN(dev_priv) < 4) {
7411                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7412
7413                 /*
7414                  * Enable double wide mode when the dot clock
7415                  * is > 90% of the (display) core speed.
7416                  */
7417                 if (intel_crtc_supports_double_wide(crtc) &&
7418                     adjusted_mode->crtc_clock > clock_limit) {
7419                         clock_limit = dev_priv->max_dotclk_freq;
7420                         pipe_config->double_wide = true;
7421                 }
7422         }
7423
7424         if (adjusted_mode->crtc_clock > clock_limit) {
7425                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7426                               adjusted_mode->crtc_clock, clock_limit,
7427                               yesno(pipe_config->double_wide));
7428                 return -EINVAL;
7429         }
7430
7431         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7432              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7433              pipe_config->base.ctm) {
7434                 /*
7435                  * There is only one pipe CSC unit per pipe, and we need that
7436                  * for output conversion from RGB->YCBCR. So if CTM is already
7437                  * applied we can't support YCBCR420 output.
7438                  */
7439                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7440                 return -EINVAL;
7441         }
7442
7443         /*
7444          * Pipe horizontal size must be even in:
7445          * - DVO ganged mode
7446          * - LVDS dual channel mode
7447          * - Double wide pipe
7448          */
7449         if (pipe_config->pipe_src_w & 1) {
7450                 if (pipe_config->double_wide) {
7451                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7452                         return -EINVAL;
7453                 }
7454
7455                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7456                     intel_is_dual_link_lvds(dev_priv)) {
7457                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7458                         return -EINVAL;
7459                 }
7460         }
7461
7462         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7463          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7464          */
7465         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7466                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7467                 return -EINVAL;
7468
7469         intel_crtc_compute_pixel_rate(pipe_config);
7470
7471         if (pipe_config->has_pch_encoder)
7472                 return ironlake_fdi_compute_config(crtc, pipe_config);
7473
7474         return 0;
7475 }
7476
7477 static void
7478 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7479 {
7480         while (*num > DATA_LINK_M_N_MASK ||
7481                *den > DATA_LINK_M_N_MASK) {
7482                 *num >>= 1;
7483                 *den >>= 1;
7484         }
7485 }
7486
7487 static void compute_m_n(unsigned int m, unsigned int n,
7488                         u32 *ret_m, u32 *ret_n,
7489                         bool constant_n)
7490 {
7491         /*
7492          * Several DP dongles in particular seem to be fussy about
7493          * too large link M/N values. Give N value as 0x8000 that
7494          * should be acceptable by specific devices. 0x8000 is the
7495          * specified fixed N value for asynchronous clock mode,
7496          * which the devices expect also in synchronous clock mode.
7497          */
7498         if (constant_n)
7499                 *ret_n = 0x8000;
7500         else
7501                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7502
7503         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7504         intel_reduce_m_n_ratio(ret_m, ret_n);
7505 }
7506
7507 void
7508 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7509                        int pixel_clock, int link_clock,
7510                        struct intel_link_m_n *m_n,
7511                        bool constant_n)
7512 {
7513         m_n->tu = 64;
7514
7515         compute_m_n(bits_per_pixel * pixel_clock,
7516                     link_clock * nlanes * 8,
7517                     &m_n->gmch_m, &m_n->gmch_n,
7518                     constant_n);
7519
7520         compute_m_n(pixel_clock, link_clock,
7521                     &m_n->link_m, &m_n->link_n,
7522                     constant_n);
7523 }
7524
7525 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7526 {
7527         if (i915_modparams.panel_use_ssc >= 0)
7528                 return i915_modparams.panel_use_ssc != 0;
7529         return dev_priv->vbt.lvds_use_ssc
7530                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7531 }
7532
7533 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7534 {
7535         return (1 << dpll->n) << 16 | dpll->m2;
7536 }
7537
7538 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7539 {
7540         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7541 }
7542
7543 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7544                                      struct intel_crtc_state *crtc_state,
7545                                      struct dpll *reduced_clock)
7546 {
7547         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7548         u32 fp, fp2 = 0;
7549
7550         if (IS_PINEVIEW(dev_priv)) {
7551                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7552                 if (reduced_clock)
7553                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7554         } else {
7555                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7556                 if (reduced_clock)
7557                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7558         }
7559
7560         crtc_state->dpll_hw_state.fp0 = fp;
7561
7562         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7563             reduced_clock) {
7564                 crtc_state->dpll_hw_state.fp1 = fp2;
7565         } else {
7566                 crtc_state->dpll_hw_state.fp1 = fp;
7567         }
7568 }
7569
7570 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7571                 pipe)
7572 {
7573         u32 reg_val;
7574
7575         /*
7576          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7577          * and set it to a reasonable value instead.
7578          */
7579         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7580         reg_val &= 0xffffff00;
7581         reg_val |= 0x00000030;
7582         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7583
7584         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7585         reg_val &= 0x00ffffff;
7586         reg_val |= 0x8c000000;
7587         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7588
7589         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7590         reg_val &= 0xffffff00;
7591         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7592
7593         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7594         reg_val &= 0x00ffffff;
7595         reg_val |= 0xb0000000;
7596         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7597 }
7598
7599 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7600                                          const struct intel_link_m_n *m_n)
7601 {
7602         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7603         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7604         enum pipe pipe = crtc->pipe;
7605
7606         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7607         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7608         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7609         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7610 }
7611
7612 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7613                                  enum transcoder transcoder)
7614 {
7615         if (IS_HASWELL(dev_priv))
7616                 return transcoder == TRANSCODER_EDP;
7617
7618         /*
7619          * Strictly speaking some registers are available before
7620          * gen7, but we only support DRRS on gen7+
7621          */
7622         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7623 }
7624
7625 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7626                                          const struct intel_link_m_n *m_n,
7627                                          const struct intel_link_m_n *m2_n2)
7628 {
7629         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7630         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7631         enum pipe pipe = crtc->pipe;
7632         enum transcoder transcoder = crtc_state->cpu_transcoder;
7633
7634         if (INTEL_GEN(dev_priv) >= 5) {
7635                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7636                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7637                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7638                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7639                 /*
7640                  *  M2_N2 registers are set only if DRRS is supported
7641                  * (to make sure the registers are not unnecessarily accessed).
7642                  */
7643                 if (m2_n2 && crtc_state->has_drrs &&
7644                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7645                         I915_WRITE(PIPE_DATA_M2(transcoder),
7646                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7647                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7648                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7649                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7650                 }
7651         } else {
7652                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7653                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7654                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7655                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7656         }
7657 }
7658
7659 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
7660 {
7661         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7662
7663         if (m_n == M1_N1) {
7664                 dp_m_n = &crtc_state->dp_m_n;
7665                 dp_m2_n2 = &crtc_state->dp_m2_n2;
7666         } else if (m_n == M2_N2) {
7667
7668                 /*
7669                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7670                  * needs to be programmed into M1_N1.
7671                  */
7672                 dp_m_n = &crtc_state->dp_m2_n2;
7673         } else {
7674                 DRM_ERROR("Unsupported divider value\n");
7675                 return;
7676         }
7677
7678         if (crtc_state->has_pch_encoder)
7679                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
7680         else
7681                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
7682 }
7683
7684 static void vlv_compute_dpll(struct intel_crtc *crtc,
7685                              struct intel_crtc_state *pipe_config)
7686 {
7687         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7688                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7689         if (crtc->pipe != PIPE_A)
7690                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7691
7692         /* DPLL not used with DSI, but still need the rest set up */
7693         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7694                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7695                         DPLL_EXT_BUFFER_ENABLE_VLV;
7696
7697         pipe_config->dpll_hw_state.dpll_md =
7698                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7699 }
7700
7701 static void chv_compute_dpll(struct intel_crtc *crtc,
7702                              struct intel_crtc_state *pipe_config)
7703 {
7704         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7705                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7706         if (crtc->pipe != PIPE_A)
7707                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7708
7709         /* DPLL not used with DSI, but still need the rest set up */
7710         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7711                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7712
7713         pipe_config->dpll_hw_state.dpll_md =
7714                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7715 }
7716
7717 static void vlv_prepare_pll(struct intel_crtc *crtc,
7718                             const struct intel_crtc_state *pipe_config)
7719 {
7720         struct drm_device *dev = crtc->base.dev;
7721         struct drm_i915_private *dev_priv = to_i915(dev);
7722         enum pipe pipe = crtc->pipe;
7723         u32 mdiv;
7724         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7725         u32 coreclk, reg_val;
7726
7727         /* Enable Refclk */
7728         I915_WRITE(DPLL(pipe),
7729                    pipe_config->dpll_hw_state.dpll &
7730                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7731
7732         /* No need to actually set up the DPLL with DSI */
7733         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7734                 return;
7735
7736         vlv_dpio_get(dev_priv);
7737
7738         bestn = pipe_config->dpll.n;
7739         bestm1 = pipe_config->dpll.m1;
7740         bestm2 = pipe_config->dpll.m2;
7741         bestp1 = pipe_config->dpll.p1;
7742         bestp2 = pipe_config->dpll.p2;
7743
7744         /* See eDP HDMI DPIO driver vbios notes doc */
7745
7746         /* PLL B needs special handling */
7747         if (pipe == PIPE_B)
7748                 vlv_pllb_recal_opamp(dev_priv, pipe);
7749
7750         /* Set up Tx target for periodic Rcomp update */
7751         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7752
7753         /* Disable target IRef on PLL */
7754         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7755         reg_val &= 0x00ffffff;
7756         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7757
7758         /* Disable fast lock */
7759         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7760
7761         /* Set idtafcrecal before PLL is enabled */
7762         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7763         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7764         mdiv |= ((bestn << DPIO_N_SHIFT));
7765         mdiv |= (1 << DPIO_K_SHIFT);
7766
7767         /*
7768          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7769          * but we don't support that).
7770          * Note: don't use the DAC post divider as it seems unstable.
7771          */
7772         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7773         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7774
7775         mdiv |= DPIO_ENABLE_CALIBRATION;
7776         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7777
7778         /* Set HBR and RBR LPF coefficients */
7779         if (pipe_config->port_clock == 162000 ||
7780             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7781             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7782                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7783                                  0x009f0003);
7784         else
7785                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7786                                  0x00d0000f);
7787
7788         if (intel_crtc_has_dp_encoder(pipe_config)) {
7789                 /* Use SSC source */
7790                 if (pipe == PIPE_A)
7791                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7792                                          0x0df40000);
7793                 else
7794                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7795                                          0x0df70000);
7796         } else { /* HDMI or VGA */
7797                 /* Use bend source */
7798                 if (pipe == PIPE_A)
7799                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7800                                          0x0df70000);
7801                 else
7802                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7803                                          0x0df40000);
7804         }
7805
7806         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7807         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7808         if (intel_crtc_has_dp_encoder(pipe_config))
7809                 coreclk |= 0x01000000;
7810         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7811
7812         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7813
7814         vlv_dpio_put(dev_priv);
7815 }
7816
7817 static void chv_prepare_pll(struct intel_crtc *crtc,
7818                             const struct intel_crtc_state *pipe_config)
7819 {
7820         struct drm_device *dev = crtc->base.dev;
7821         struct drm_i915_private *dev_priv = to_i915(dev);
7822         enum pipe pipe = crtc->pipe;
7823         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7824         u32 loopfilter, tribuf_calcntr;
7825         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7826         u32 dpio_val;
7827         int vco;
7828
7829         /* Enable Refclk and SSC */
7830         I915_WRITE(DPLL(pipe),
7831                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7832
7833         /* No need to actually set up the DPLL with DSI */
7834         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7835                 return;
7836
7837         bestn = pipe_config->dpll.n;
7838         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7839         bestm1 = pipe_config->dpll.m1;
7840         bestm2 = pipe_config->dpll.m2 >> 22;
7841         bestp1 = pipe_config->dpll.p1;
7842         bestp2 = pipe_config->dpll.p2;
7843         vco = pipe_config->dpll.vco;
7844         dpio_val = 0;
7845         loopfilter = 0;
7846
7847         vlv_dpio_get(dev_priv);
7848
7849         /* p1 and p2 divider */
7850         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7851                         5 << DPIO_CHV_S1_DIV_SHIFT |
7852                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7853                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7854                         1 << DPIO_CHV_K_DIV_SHIFT);
7855
7856         /* Feedback post-divider - m2 */
7857         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7858
7859         /* Feedback refclk divider - n and m1 */
7860         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7861                         DPIO_CHV_M1_DIV_BY_2 |
7862                         1 << DPIO_CHV_N_DIV_SHIFT);
7863
7864         /* M2 fraction division */
7865         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7866
7867         /* M2 fraction division enable */
7868         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7869         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7870         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7871         if (bestm2_frac)
7872                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7873         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7874
7875         /* Program digital lock detect threshold */
7876         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7877         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7878                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7879         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7880         if (!bestm2_frac)
7881                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7882         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7883
7884         /* Loop filter */
7885         if (vco == 5400000) {
7886                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7887                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7888                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7889                 tribuf_calcntr = 0x9;
7890         } else if (vco <= 6200000) {
7891                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7892                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7893                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7894                 tribuf_calcntr = 0x9;
7895         } else if (vco <= 6480000) {
7896                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7897                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7898                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7899                 tribuf_calcntr = 0x8;
7900         } else {
7901                 /* Not supported. Apply the same limits as in the max case */
7902                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7903                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7904                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7905                 tribuf_calcntr = 0;
7906         }
7907         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7908
7909         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7910         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7911         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7912         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7913
7914         /* AFC Recal */
7915         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7916                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7917                         DPIO_AFC_RECAL);
7918
7919         vlv_dpio_put(dev_priv);
7920 }
7921
7922 /**
7923  * vlv_force_pll_on - forcibly enable just the PLL
7924  * @dev_priv: i915 private structure
7925  * @pipe: pipe PLL to enable
7926  * @dpll: PLL configuration
7927  *
7928  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7929  * in cases where we need the PLL enabled even when @pipe is not going to
7930  * be enabled.
7931  */
7932 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7933                      const struct dpll *dpll)
7934 {
7935         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7936         struct intel_crtc_state *pipe_config;
7937
7938         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7939         if (!pipe_config)
7940                 return -ENOMEM;
7941
7942         pipe_config->base.crtc = &crtc->base;
7943         pipe_config->pixel_multiplier = 1;
7944         pipe_config->dpll = *dpll;
7945
7946         if (IS_CHERRYVIEW(dev_priv)) {
7947                 chv_compute_dpll(crtc, pipe_config);
7948                 chv_prepare_pll(crtc, pipe_config);
7949                 chv_enable_pll(crtc, pipe_config);
7950         } else {
7951                 vlv_compute_dpll(crtc, pipe_config);
7952                 vlv_prepare_pll(crtc, pipe_config);
7953                 vlv_enable_pll(crtc, pipe_config);
7954         }
7955
7956         kfree(pipe_config);
7957
7958         return 0;
7959 }
7960
7961 /**
7962  * vlv_force_pll_off - forcibly disable just the PLL
7963  * @dev_priv: i915 private structure
7964  * @pipe: pipe PLL to disable
7965  *
7966  * Disable the PLL for @pipe. To be used in cases where we need
7967  * the PLL enabled even when @pipe is not going to be enabled.
7968  */
7969 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7970 {
7971         if (IS_CHERRYVIEW(dev_priv))
7972                 chv_disable_pll(dev_priv, pipe);
7973         else
7974                 vlv_disable_pll(dev_priv, pipe);
7975 }
7976
7977 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7978                               struct intel_crtc_state *crtc_state,
7979                               struct dpll *reduced_clock)
7980 {
7981         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7982         u32 dpll;
7983         struct dpll *clock = &crtc_state->dpll;
7984
7985         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7986
7987         dpll = DPLL_VGA_MODE_DIS;
7988
7989         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7990                 dpll |= DPLLB_MODE_LVDS;
7991         else
7992                 dpll |= DPLLB_MODE_DAC_SERIAL;
7993
7994         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7995             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7996                 dpll |= (crtc_state->pixel_multiplier - 1)
7997                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7998         }
7999
8000         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8001             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8002                 dpll |= DPLL_SDVO_HIGH_SPEED;
8003
8004         if (intel_crtc_has_dp_encoder(crtc_state))
8005                 dpll |= DPLL_SDVO_HIGH_SPEED;
8006
8007         /* compute bitmask from p1 value */
8008         if (IS_PINEVIEW(dev_priv))
8009                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8010         else {
8011                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8012                 if (IS_G4X(dev_priv) && reduced_clock)
8013                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8014         }
8015         switch (clock->p2) {
8016         case 5:
8017                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8018                 break;
8019         case 7:
8020                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8021                 break;
8022         case 10:
8023                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8024                 break;
8025         case 14:
8026                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8027                 break;
8028         }
8029         if (INTEL_GEN(dev_priv) >= 4)
8030                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8031
8032         if (crtc_state->sdvo_tv_clock)
8033                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8034         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8035                  intel_panel_use_ssc(dev_priv))
8036                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8037         else
8038                 dpll |= PLL_REF_INPUT_DREFCLK;
8039
8040         dpll |= DPLL_VCO_ENABLE;
8041         crtc_state->dpll_hw_state.dpll = dpll;
8042
8043         if (INTEL_GEN(dev_priv) >= 4) {
8044                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8045                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8046                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8047         }
8048 }
8049
8050 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8051                               struct intel_crtc_state *crtc_state,
8052                               struct dpll *reduced_clock)
8053 {
8054         struct drm_device *dev = crtc->base.dev;
8055         struct drm_i915_private *dev_priv = to_i915(dev);
8056         u32 dpll;
8057         struct dpll *clock = &crtc_state->dpll;
8058
8059         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8060
8061         dpll = DPLL_VGA_MODE_DIS;
8062
8063         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8064                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8065         } else {
8066                 if (clock->p1 == 2)
8067                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8068                 else
8069                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8070                 if (clock->p2 == 4)
8071                         dpll |= PLL_P2_DIVIDE_BY_4;
8072         }
8073
8074         /*
8075          * Bspec:
8076          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8077          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8078          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8079          *  Enable) must be set to “1” in both the DPLL A Control Register
8080          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8081          *
8082          * For simplicity We simply keep both bits always enabled in
8083          * both DPLLS. The spec says we should disable the DVO 2X clock
8084          * when not needed, but this seems to work fine in practice.
8085          */
8086         if (IS_I830(dev_priv) ||
8087             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8088                 dpll |= DPLL_DVO_2X_MODE;
8089
8090         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8091             intel_panel_use_ssc(dev_priv))
8092                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8093         else
8094                 dpll |= PLL_REF_INPUT_DREFCLK;
8095
8096         dpll |= DPLL_VCO_ENABLE;
8097         crtc_state->dpll_hw_state.dpll = dpll;
8098 }
8099
8100 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8101 {
8102         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8103         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8104         enum pipe pipe = crtc->pipe;
8105         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8106         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
8107         u32 crtc_vtotal, crtc_vblank_end;
8108         int vsyncshift = 0;
8109
8110         /* We need to be careful not to changed the adjusted mode, for otherwise
8111          * the hw state checker will get angry at the mismatch. */
8112         crtc_vtotal = adjusted_mode->crtc_vtotal;
8113         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8114
8115         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8116                 /* the chip adds 2 halflines automatically */
8117                 crtc_vtotal -= 1;
8118                 crtc_vblank_end -= 1;
8119
8120                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8121                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8122                 else
8123                         vsyncshift = adjusted_mode->crtc_hsync_start -
8124                                 adjusted_mode->crtc_htotal / 2;
8125                 if (vsyncshift < 0)
8126                         vsyncshift += adjusted_mode->crtc_htotal;
8127         }
8128
8129         if (INTEL_GEN(dev_priv) > 3)
8130                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8131
8132         I915_WRITE(HTOTAL(cpu_transcoder),
8133                    (adjusted_mode->crtc_hdisplay - 1) |
8134                    ((adjusted_mode->crtc_htotal - 1) << 16));
8135         I915_WRITE(HBLANK(cpu_transcoder),
8136                    (adjusted_mode->crtc_hblank_start - 1) |
8137                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8138         I915_WRITE(HSYNC(cpu_transcoder),
8139                    (adjusted_mode->crtc_hsync_start - 1) |
8140                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8141
8142         I915_WRITE(VTOTAL(cpu_transcoder),
8143                    (adjusted_mode->crtc_vdisplay - 1) |
8144                    ((crtc_vtotal - 1) << 16));
8145         I915_WRITE(VBLANK(cpu_transcoder),
8146                    (adjusted_mode->crtc_vblank_start - 1) |
8147                    ((crtc_vblank_end - 1) << 16));
8148         I915_WRITE(VSYNC(cpu_transcoder),
8149                    (adjusted_mode->crtc_vsync_start - 1) |
8150                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8151
8152         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8153          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8154          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8155          * bits. */
8156         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8157             (pipe == PIPE_B || pipe == PIPE_C))
8158                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8159
8160 }
8161
8162 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8163 {
8164         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8165         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8166         enum pipe pipe = crtc->pipe;
8167
8168         /* pipesrc controls the size that is scaled from, which should
8169          * always be the user's requested size.
8170          */
8171         I915_WRITE(PIPESRC(pipe),
8172                    ((crtc_state->pipe_src_w - 1) << 16) |
8173                    (crtc_state->pipe_src_h - 1));
8174 }
8175
8176 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8177                                    struct intel_crtc_state *pipe_config)
8178 {
8179         struct drm_device *dev = crtc->base.dev;
8180         struct drm_i915_private *dev_priv = to_i915(dev);
8181         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8182         u32 tmp;
8183
8184         tmp = I915_READ(HTOTAL(cpu_transcoder));
8185         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8186         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8187
8188         if (!transcoder_is_dsi(cpu_transcoder)) {
8189                 tmp = I915_READ(HBLANK(cpu_transcoder));
8190                 pipe_config->base.adjusted_mode.crtc_hblank_start =
8191                                                         (tmp & 0xffff) + 1;
8192                 pipe_config->base.adjusted_mode.crtc_hblank_end =
8193                                                 ((tmp >> 16) & 0xffff) + 1;
8194         }
8195         tmp = I915_READ(HSYNC(cpu_transcoder));
8196         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8197         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8198
8199         tmp = I915_READ(VTOTAL(cpu_transcoder));
8200         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8201         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8202
8203         if (!transcoder_is_dsi(cpu_transcoder)) {
8204                 tmp = I915_READ(VBLANK(cpu_transcoder));
8205                 pipe_config->base.adjusted_mode.crtc_vblank_start =
8206                                                         (tmp & 0xffff) + 1;
8207                 pipe_config->base.adjusted_mode.crtc_vblank_end =
8208                                                 ((tmp >> 16) & 0xffff) + 1;
8209         }
8210         tmp = I915_READ(VSYNC(cpu_transcoder));
8211         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8212         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8213
8214         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
8215                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8216                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
8217                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
8218         }
8219 }
8220
8221 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8222                                     struct intel_crtc_state *pipe_config)
8223 {
8224         struct drm_device *dev = crtc->base.dev;
8225         struct drm_i915_private *dev_priv = to_i915(dev);
8226         u32 tmp;
8227
8228         tmp = I915_READ(PIPESRC(crtc->pipe));
8229         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8230         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8231
8232         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
8233         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
8234 }
8235
8236 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8237                                  struct intel_crtc_state *pipe_config)
8238 {
8239         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
8240         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
8241         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
8242         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
8243
8244         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
8245         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
8246         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
8247         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
8248
8249         mode->flags = pipe_config->base.adjusted_mode.flags;
8250         mode->type = DRM_MODE_TYPE_DRIVER;
8251
8252         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
8253
8254         mode->hsync = drm_mode_hsync(mode);
8255         mode->vrefresh = drm_mode_vrefresh(mode);
8256         drm_mode_set_name(mode);
8257 }
8258
8259 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8260 {
8261         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8262         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8263         u32 pipeconf;
8264
8265         pipeconf = 0;
8266
8267         /* we keep both pipes enabled on 830 */
8268         if (IS_I830(dev_priv))
8269                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8270
8271         if (crtc_state->double_wide)
8272                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8273
8274         /* only g4x and later have fancy bpc/dither controls */
8275         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8276             IS_CHERRYVIEW(dev_priv)) {
8277                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8278                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8279                         pipeconf |= PIPECONF_DITHER_EN |
8280                                     PIPECONF_DITHER_TYPE_SP;
8281
8282                 switch (crtc_state->pipe_bpp) {
8283                 case 18:
8284                         pipeconf |= PIPECONF_6BPC;
8285                         break;
8286                 case 24:
8287                         pipeconf |= PIPECONF_8BPC;
8288                         break;
8289                 case 30:
8290                         pipeconf |= PIPECONF_10BPC;
8291                         break;
8292                 default:
8293                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8294                         BUG();
8295                 }
8296         }
8297
8298         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8299                 if (INTEL_GEN(dev_priv) < 4 ||
8300                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8301                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8302                 else
8303                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8304         } else {
8305                 pipeconf |= PIPECONF_PROGRESSIVE;
8306         }
8307
8308         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8309              crtc_state->limited_color_range)
8310                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8311
8312         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8313
8314         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8315         POSTING_READ(PIPECONF(crtc->pipe));
8316 }
8317
8318 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8319                                    struct intel_crtc_state *crtc_state)
8320 {
8321         struct drm_device *dev = crtc->base.dev;
8322         struct drm_i915_private *dev_priv = to_i915(dev);
8323         const struct intel_limit *limit;
8324         int refclk = 48000;
8325
8326         memset(&crtc_state->dpll_hw_state, 0,
8327                sizeof(crtc_state->dpll_hw_state));
8328
8329         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8330                 if (intel_panel_use_ssc(dev_priv)) {
8331                         refclk = dev_priv->vbt.lvds_ssc_freq;
8332                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8333                 }
8334
8335                 limit = &intel_limits_i8xx_lvds;
8336         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8337                 limit = &intel_limits_i8xx_dvo;
8338         } else {
8339                 limit = &intel_limits_i8xx_dac;
8340         }
8341
8342         if (!crtc_state->clock_set &&
8343             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8344                                  refclk, NULL, &crtc_state->dpll)) {
8345                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8346                 return -EINVAL;
8347         }
8348
8349         i8xx_compute_dpll(crtc, crtc_state, NULL);
8350
8351         return 0;
8352 }
8353
8354 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8355                                   struct intel_crtc_state *crtc_state)
8356 {
8357         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8358         const struct intel_limit *limit;
8359         int refclk = 96000;
8360
8361         memset(&crtc_state->dpll_hw_state, 0,
8362                sizeof(crtc_state->dpll_hw_state));
8363
8364         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8365                 if (intel_panel_use_ssc(dev_priv)) {
8366                         refclk = dev_priv->vbt.lvds_ssc_freq;
8367                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8368                 }
8369
8370                 if (intel_is_dual_link_lvds(dev_priv))
8371                         limit = &intel_limits_g4x_dual_channel_lvds;
8372                 else
8373                         limit = &intel_limits_g4x_single_channel_lvds;
8374         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8375                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8376                 limit = &intel_limits_g4x_hdmi;
8377         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8378                 limit = &intel_limits_g4x_sdvo;
8379         } else {
8380                 /* The option is for other outputs */
8381                 limit = &intel_limits_i9xx_sdvo;
8382         }
8383
8384         if (!crtc_state->clock_set &&
8385             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8386                                 refclk, NULL, &crtc_state->dpll)) {
8387                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8388                 return -EINVAL;
8389         }
8390
8391         i9xx_compute_dpll(crtc, crtc_state, NULL);
8392
8393         return 0;
8394 }
8395
8396 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8397                                   struct intel_crtc_state *crtc_state)
8398 {
8399         struct drm_device *dev = crtc->base.dev;
8400         struct drm_i915_private *dev_priv = to_i915(dev);
8401         const struct intel_limit *limit;
8402         int refclk = 96000;
8403
8404         memset(&crtc_state->dpll_hw_state, 0,
8405                sizeof(crtc_state->dpll_hw_state));
8406
8407         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8408                 if (intel_panel_use_ssc(dev_priv)) {
8409                         refclk = dev_priv->vbt.lvds_ssc_freq;
8410                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8411                 }
8412
8413                 limit = &intel_limits_pineview_lvds;
8414         } else {
8415                 limit = &intel_limits_pineview_sdvo;
8416         }
8417
8418         if (!crtc_state->clock_set &&
8419             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8420                                 refclk, NULL, &crtc_state->dpll)) {
8421                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8422                 return -EINVAL;
8423         }
8424
8425         i9xx_compute_dpll(crtc, crtc_state, NULL);
8426
8427         return 0;
8428 }
8429
8430 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8431                                    struct intel_crtc_state *crtc_state)
8432 {
8433         struct drm_device *dev = crtc->base.dev;
8434         struct drm_i915_private *dev_priv = to_i915(dev);
8435         const struct intel_limit *limit;
8436         int refclk = 96000;
8437
8438         memset(&crtc_state->dpll_hw_state, 0,
8439                sizeof(crtc_state->dpll_hw_state));
8440
8441         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8442                 if (intel_panel_use_ssc(dev_priv)) {
8443                         refclk = dev_priv->vbt.lvds_ssc_freq;
8444                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8445                 }
8446
8447                 limit = &intel_limits_i9xx_lvds;
8448         } else {
8449                 limit = &intel_limits_i9xx_sdvo;
8450         }
8451
8452         if (!crtc_state->clock_set &&
8453             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8454                                  refclk, NULL, &crtc_state->dpll)) {
8455                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8456                 return -EINVAL;
8457         }
8458
8459         i9xx_compute_dpll(crtc, crtc_state, NULL);
8460
8461         return 0;
8462 }
8463
8464 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8465                                   struct intel_crtc_state *crtc_state)
8466 {
8467         int refclk = 100000;
8468         const struct intel_limit *limit = &intel_limits_chv;
8469
8470         memset(&crtc_state->dpll_hw_state, 0,
8471                sizeof(crtc_state->dpll_hw_state));
8472
8473         if (!crtc_state->clock_set &&
8474             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8475                                 refclk, NULL, &crtc_state->dpll)) {
8476                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8477                 return -EINVAL;
8478         }
8479
8480         chv_compute_dpll(crtc, crtc_state);
8481
8482         return 0;
8483 }
8484
8485 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8486                                   struct intel_crtc_state *crtc_state)
8487 {
8488         int refclk = 100000;
8489         const struct intel_limit *limit = &intel_limits_vlv;
8490
8491         memset(&crtc_state->dpll_hw_state, 0,
8492                sizeof(crtc_state->dpll_hw_state));
8493
8494         if (!crtc_state->clock_set &&
8495             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8496                                 refclk, NULL, &crtc_state->dpll)) {
8497                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8498                 return -EINVAL;
8499         }
8500
8501         vlv_compute_dpll(crtc, crtc_state);
8502
8503         return 0;
8504 }
8505
8506 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8507 {
8508         if (IS_I830(dev_priv))
8509                 return false;
8510
8511         return INTEL_GEN(dev_priv) >= 4 ||
8512                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8513 }
8514
8515 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8516                                  struct intel_crtc_state *pipe_config)
8517 {
8518         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8519         u32 tmp;
8520
8521         if (!i9xx_has_pfit(dev_priv))
8522                 return;
8523
8524         tmp = I915_READ(PFIT_CONTROL);
8525         if (!(tmp & PFIT_ENABLE))
8526                 return;
8527
8528         /* Check whether the pfit is attached to our pipe. */
8529         if (INTEL_GEN(dev_priv) < 4) {
8530                 if (crtc->pipe != PIPE_B)
8531                         return;
8532         } else {
8533                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8534                         return;
8535         }
8536
8537         pipe_config->gmch_pfit.control = tmp;
8538         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8539 }
8540
8541 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8542                                struct intel_crtc_state *pipe_config)
8543 {
8544         struct drm_device *dev = crtc->base.dev;
8545         struct drm_i915_private *dev_priv = to_i915(dev);
8546         enum pipe pipe = crtc->pipe;
8547         struct dpll clock;
8548         u32 mdiv;
8549         int refclk = 100000;
8550
8551         /* In case of DSI, DPLL will not be used */
8552         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8553                 return;
8554
8555         vlv_dpio_get(dev_priv);
8556         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8557         vlv_dpio_put(dev_priv);
8558
8559         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8560         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8561         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8562         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8563         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8564
8565         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8566 }
8567
8568 static void
8569 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8570                               struct intel_initial_plane_config *plane_config)
8571 {
8572         struct drm_device *dev = crtc->base.dev;
8573         struct drm_i915_private *dev_priv = to_i915(dev);
8574         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8575         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8576         enum pipe pipe;
8577         u32 val, base, offset;
8578         int fourcc, pixel_format;
8579         unsigned int aligned_height;
8580         struct drm_framebuffer *fb;
8581         struct intel_framebuffer *intel_fb;
8582
8583         if (!plane->get_hw_state(plane, &pipe))
8584                 return;
8585
8586         WARN_ON(pipe != crtc->pipe);
8587
8588         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8589         if (!intel_fb) {
8590                 DRM_DEBUG_KMS("failed to alloc fb\n");
8591                 return;
8592         }
8593
8594         fb = &intel_fb->base;
8595
8596         fb->dev = dev;
8597
8598         val = I915_READ(DSPCNTR(i9xx_plane));
8599
8600         if (INTEL_GEN(dev_priv) >= 4) {
8601                 if (val & DISPPLANE_TILED) {
8602                         plane_config->tiling = I915_TILING_X;
8603                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8604                 }
8605
8606                 if (val & DISPPLANE_ROTATE_180)
8607                         plane_config->rotation = DRM_MODE_ROTATE_180;
8608         }
8609
8610         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8611             val & DISPPLANE_MIRROR)
8612                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8613
8614         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8615         fourcc = i9xx_format_to_fourcc(pixel_format);
8616         fb->format = drm_format_info(fourcc);
8617
8618         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8619                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8620                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8621         } else if (INTEL_GEN(dev_priv) >= 4) {
8622                 if (plane_config->tiling)
8623                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8624                 else
8625                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8626                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8627         } else {
8628                 base = I915_READ(DSPADDR(i9xx_plane));
8629         }
8630         plane_config->base = base;
8631
8632         val = I915_READ(PIPESRC(pipe));
8633         fb->width = ((val >> 16) & 0xfff) + 1;
8634         fb->height = ((val >> 0) & 0xfff) + 1;
8635
8636         val = I915_READ(DSPSTRIDE(i9xx_plane));
8637         fb->pitches[0] = val & 0xffffffc0;
8638
8639         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8640
8641         plane_config->size = fb->pitches[0] * aligned_height;
8642
8643         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8644                       crtc->base.name, plane->base.name, fb->width, fb->height,
8645                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8646                       plane_config->size);
8647
8648         plane_config->fb = intel_fb;
8649 }
8650
8651 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8652                                struct intel_crtc_state *pipe_config)
8653 {
8654         struct drm_device *dev = crtc->base.dev;
8655         struct drm_i915_private *dev_priv = to_i915(dev);
8656         enum pipe pipe = crtc->pipe;
8657         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8658         struct dpll clock;
8659         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8660         int refclk = 100000;
8661
8662         /* In case of DSI, DPLL will not be used */
8663         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8664                 return;
8665
8666         vlv_dpio_get(dev_priv);
8667         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8668         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8669         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8670         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8671         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8672         vlv_dpio_put(dev_priv);
8673
8674         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8675         clock.m2 = (pll_dw0 & 0xff) << 22;
8676         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8677                 clock.m2 |= pll_dw2 & 0x3fffff;
8678         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8679         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8680         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8681
8682         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8683 }
8684
8685 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
8686                                         struct intel_crtc_state *pipe_config)
8687 {
8688         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8689         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
8690
8691         pipe_config->lspcon_downsampling = false;
8692
8693         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8694                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
8695
8696                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
8697                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
8698                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
8699
8700                         if (ycbcr420_enabled) {
8701                                 /* We support 4:2:0 in full blend mode only */
8702                                 if (!blend)
8703                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8704                                 else if (!(IS_GEMINILAKE(dev_priv) ||
8705                                            INTEL_GEN(dev_priv) >= 10))
8706                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8707                                 else
8708                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
8709                         } else {
8710                                 /*
8711                                  * Currently there is no interface defined to
8712                                  * check user preference between RGB/YCBCR444
8713                                  * or YCBCR420. So the only possible case for
8714                                  * YCBCR444 usage is driving YCBCR420 output
8715                                  * with LSPCON, when pipe is configured for
8716                                  * YCBCR444 output and LSPCON takes care of
8717                                  * downsampling it.
8718                                  */
8719                                 pipe_config->lspcon_downsampling = true;
8720                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
8721                         }
8722                 }
8723         }
8724
8725         pipe_config->output_format = output;
8726 }
8727
8728 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
8729 {
8730         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8731         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8732         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8733         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8734         u32 tmp;
8735
8736         tmp = I915_READ(DSPCNTR(i9xx_plane));
8737
8738         if (tmp & DISPPLANE_GAMMA_ENABLE)
8739                 crtc_state->gamma_enable = true;
8740
8741         if (!HAS_GMCH(dev_priv) &&
8742             tmp & DISPPLANE_PIPE_CSC_ENABLE)
8743                 crtc_state->csc_enable = true;
8744 }
8745
8746 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8747                                  struct intel_crtc_state *pipe_config)
8748 {
8749         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8750         enum intel_display_power_domain power_domain;
8751         intel_wakeref_t wakeref;
8752         u32 tmp;
8753         bool ret;
8754
8755         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8756         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
8757         if (!wakeref)
8758                 return false;
8759
8760         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
8761         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8762         pipe_config->shared_dpll = NULL;
8763
8764         ret = false;
8765
8766         tmp = I915_READ(PIPECONF(crtc->pipe));
8767         if (!(tmp & PIPECONF_ENABLE))
8768                 goto out;
8769
8770         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8771             IS_CHERRYVIEW(dev_priv)) {
8772                 switch (tmp & PIPECONF_BPC_MASK) {
8773                 case PIPECONF_6BPC:
8774                         pipe_config->pipe_bpp = 18;
8775                         break;
8776                 case PIPECONF_8BPC:
8777                         pipe_config->pipe_bpp = 24;
8778                         break;
8779                 case PIPECONF_10BPC:
8780                         pipe_config->pipe_bpp = 30;
8781                         break;
8782                 default:
8783                         break;
8784                 }
8785         }
8786
8787         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8788             (tmp & PIPECONF_COLOR_RANGE_SELECT))
8789                 pipe_config->limited_color_range = true;
8790
8791         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
8792                 PIPECONF_GAMMA_MODE_SHIFT;
8793
8794         if (IS_CHERRYVIEW(dev_priv))
8795                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
8796
8797         i9xx_get_pipe_color_config(pipe_config);
8798         intel_color_get_config(pipe_config);
8799
8800         if (INTEL_GEN(dev_priv) < 4)
8801                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8802
8803         intel_get_pipe_timings(crtc, pipe_config);
8804         intel_get_pipe_src_size(crtc, pipe_config);
8805
8806         i9xx_get_pfit_config(crtc, pipe_config);
8807
8808         if (INTEL_GEN(dev_priv) >= 4) {
8809                 /* No way to read it out on pipes B and C */
8810                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8811                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8812                 else
8813                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8814                 pipe_config->pixel_multiplier =
8815                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8816                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8817                 pipe_config->dpll_hw_state.dpll_md = tmp;
8818         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8819                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8820                 tmp = I915_READ(DPLL(crtc->pipe));
8821                 pipe_config->pixel_multiplier =
8822                         ((tmp & SDVO_MULTIPLIER_MASK)
8823                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8824         } else {
8825                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8826                  * port and will be fixed up in the encoder->get_config
8827                  * function. */
8828                 pipe_config->pixel_multiplier = 1;
8829         }
8830         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8831         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8832                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8833                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8834         } else {
8835                 /* Mask out read-only status bits. */
8836                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8837                                                      DPLL_PORTC_READY_MASK |
8838                                                      DPLL_PORTB_READY_MASK);
8839         }
8840
8841         if (IS_CHERRYVIEW(dev_priv))
8842                 chv_crtc_clock_get(crtc, pipe_config);
8843         else if (IS_VALLEYVIEW(dev_priv))
8844                 vlv_crtc_clock_get(crtc, pipe_config);
8845         else
8846                 i9xx_crtc_clock_get(crtc, pipe_config);
8847
8848         /*
8849          * Normally the dotclock is filled in by the encoder .get_config()
8850          * but in case the pipe is enabled w/o any ports we need a sane
8851          * default.
8852          */
8853         pipe_config->base.adjusted_mode.crtc_clock =
8854                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8855
8856         ret = true;
8857
8858 out:
8859         intel_display_power_put(dev_priv, power_domain, wakeref);
8860
8861         return ret;
8862 }
8863
8864 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8865 {
8866         struct intel_encoder *encoder;
8867         int i;
8868         u32 val, final;
8869         bool has_lvds = false;
8870         bool has_cpu_edp = false;
8871         bool has_panel = false;
8872         bool has_ck505 = false;
8873         bool can_ssc = false;
8874         bool using_ssc_source = false;
8875
8876         /* We need to take the global config into account */
8877         for_each_intel_encoder(&dev_priv->drm, encoder) {
8878                 switch (encoder->type) {
8879                 case INTEL_OUTPUT_LVDS:
8880                         has_panel = true;
8881                         has_lvds = true;
8882                         break;
8883                 case INTEL_OUTPUT_EDP:
8884                         has_panel = true;
8885                         if (encoder->port == PORT_A)
8886                                 has_cpu_edp = true;
8887                         break;
8888                 default:
8889                         break;
8890                 }
8891         }
8892
8893         if (HAS_PCH_IBX(dev_priv)) {
8894                 has_ck505 = dev_priv->vbt.display_clock_mode;
8895                 can_ssc = has_ck505;
8896         } else {
8897                 has_ck505 = false;
8898                 can_ssc = true;
8899         }
8900
8901         /* Check if any DPLLs are using the SSC source */
8902         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8903                 u32 temp = I915_READ(PCH_DPLL(i));
8904
8905                 if (!(temp & DPLL_VCO_ENABLE))
8906                         continue;
8907
8908                 if ((temp & PLL_REF_INPUT_MASK) ==
8909                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8910                         using_ssc_source = true;
8911                         break;
8912                 }
8913         }
8914
8915         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8916                       has_panel, has_lvds, has_ck505, using_ssc_source);
8917
8918         /* Ironlake: try to setup display ref clock before DPLL
8919          * enabling. This is only under driver's control after
8920          * PCH B stepping, previous chipset stepping should be
8921          * ignoring this setting.
8922          */
8923         val = I915_READ(PCH_DREF_CONTROL);
8924
8925         /* As we must carefully and slowly disable/enable each source in turn,
8926          * compute the final state we want first and check if we need to
8927          * make any changes at all.
8928          */
8929         final = val;
8930         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8931         if (has_ck505)
8932                 final |= DREF_NONSPREAD_CK505_ENABLE;
8933         else
8934                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8935
8936         final &= ~DREF_SSC_SOURCE_MASK;
8937         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8938         final &= ~DREF_SSC1_ENABLE;
8939
8940         if (has_panel) {
8941                 final |= DREF_SSC_SOURCE_ENABLE;
8942
8943                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8944                         final |= DREF_SSC1_ENABLE;
8945
8946                 if (has_cpu_edp) {
8947                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8948                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8949                         else
8950                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8951                 } else
8952                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8953         } else if (using_ssc_source) {
8954                 final |= DREF_SSC_SOURCE_ENABLE;
8955                 final |= DREF_SSC1_ENABLE;
8956         }
8957
8958         if (final == val)
8959                 return;
8960
8961         /* Always enable nonspread source */
8962         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8963
8964         if (has_ck505)
8965                 val |= DREF_NONSPREAD_CK505_ENABLE;
8966         else
8967                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8968
8969         if (has_panel) {
8970                 val &= ~DREF_SSC_SOURCE_MASK;
8971                 val |= DREF_SSC_SOURCE_ENABLE;
8972
8973                 /* SSC must be turned on before enabling the CPU output  */
8974                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8975                         DRM_DEBUG_KMS("Using SSC on panel\n");
8976                         val |= DREF_SSC1_ENABLE;
8977                 } else
8978                         val &= ~DREF_SSC1_ENABLE;
8979
8980                 /* Get SSC going before enabling the outputs */
8981                 I915_WRITE(PCH_DREF_CONTROL, val);
8982                 POSTING_READ(PCH_DREF_CONTROL);
8983                 udelay(200);
8984
8985                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8986
8987                 /* Enable CPU source on CPU attached eDP */
8988                 if (has_cpu_edp) {
8989                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8990                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8991                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8992                         } else
8993                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8994                 } else
8995                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8996
8997                 I915_WRITE(PCH_DREF_CONTROL, val);
8998                 POSTING_READ(PCH_DREF_CONTROL);
8999                 udelay(200);
9000         } else {
9001                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9002
9003                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9004
9005                 /* Turn off CPU output */
9006                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9007
9008                 I915_WRITE(PCH_DREF_CONTROL, val);
9009                 POSTING_READ(PCH_DREF_CONTROL);
9010                 udelay(200);
9011
9012                 if (!using_ssc_source) {
9013                         DRM_DEBUG_KMS("Disabling SSC source\n");
9014
9015                         /* Turn off the SSC source */
9016                         val &= ~DREF_SSC_SOURCE_MASK;
9017                         val |= DREF_SSC_SOURCE_DISABLE;
9018
9019                         /* Turn off SSC1 */
9020                         val &= ~DREF_SSC1_ENABLE;
9021
9022                         I915_WRITE(PCH_DREF_CONTROL, val);
9023                         POSTING_READ(PCH_DREF_CONTROL);
9024                         udelay(200);
9025                 }
9026         }
9027
9028         BUG_ON(val != final);
9029 }
9030
9031 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9032 {
9033         u32 tmp;
9034
9035         tmp = I915_READ(SOUTH_CHICKEN2);
9036         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9037         I915_WRITE(SOUTH_CHICKEN2, tmp);
9038
9039         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9040                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9041                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9042
9043         tmp = I915_READ(SOUTH_CHICKEN2);
9044         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9045         I915_WRITE(SOUTH_CHICKEN2, tmp);
9046
9047         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9048                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9049                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9050 }
9051
9052 /* WaMPhyProgramming:hsw */
9053 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9054 {
9055         u32 tmp;
9056
9057         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9058         tmp &= ~(0xFF << 24);
9059         tmp |= (0x12 << 24);
9060         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9061
9062         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9063         tmp |= (1 << 11);
9064         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9065
9066         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9067         tmp |= (1 << 11);
9068         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9069
9070         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9071         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9072         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9073
9074         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9075         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9076         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9077
9078         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9079         tmp &= ~(7 << 13);
9080         tmp |= (5 << 13);
9081         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9082
9083         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9084         tmp &= ~(7 << 13);
9085         tmp |= (5 << 13);
9086         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9087
9088         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9089         tmp &= ~0xFF;
9090         tmp |= 0x1C;
9091         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9092
9093         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9094         tmp &= ~0xFF;
9095         tmp |= 0x1C;
9096         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9097
9098         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9099         tmp &= ~(0xFF << 16);
9100         tmp |= (0x1C << 16);
9101         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9102
9103         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9104         tmp &= ~(0xFF << 16);
9105         tmp |= (0x1C << 16);
9106         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9107
9108         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9109         tmp |= (1 << 27);
9110         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9111
9112         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9113         tmp |= (1 << 27);
9114         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9115
9116         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9117         tmp &= ~(0xF << 28);
9118         tmp |= (4 << 28);
9119         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9120
9121         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9122         tmp &= ~(0xF << 28);
9123         tmp |= (4 << 28);
9124         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9125 }
9126
9127 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9128  * Programming" based on the parameters passed:
9129  * - Sequence to enable CLKOUT_DP
9130  * - Sequence to enable CLKOUT_DP without spread
9131  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9132  */
9133 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9134                                  bool with_spread, bool with_fdi)
9135 {
9136         u32 reg, tmp;
9137
9138         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9139                 with_spread = true;
9140         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9141             with_fdi, "LP PCH doesn't have FDI\n"))
9142                 with_fdi = false;
9143
9144         mutex_lock(&dev_priv->sb_lock);
9145
9146         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9147         tmp &= ~SBI_SSCCTL_DISABLE;
9148         tmp |= SBI_SSCCTL_PATHALT;
9149         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9150
9151         udelay(24);
9152
9153         if (with_spread) {
9154                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9155                 tmp &= ~SBI_SSCCTL_PATHALT;
9156                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9157
9158                 if (with_fdi) {
9159                         lpt_reset_fdi_mphy(dev_priv);
9160                         lpt_program_fdi_mphy(dev_priv);
9161                 }
9162         }
9163
9164         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9165         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9166         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9167         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9168
9169         mutex_unlock(&dev_priv->sb_lock);
9170 }
9171
9172 /* Sequence to disable CLKOUT_DP */
9173 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9174 {
9175         u32 reg, tmp;
9176
9177         mutex_lock(&dev_priv->sb_lock);
9178
9179         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9180         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9181         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9182         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9183
9184         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9185         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9186                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9187                         tmp |= SBI_SSCCTL_PATHALT;
9188                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9189                         udelay(32);
9190                 }
9191                 tmp |= SBI_SSCCTL_DISABLE;
9192                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9193         }
9194
9195         mutex_unlock(&dev_priv->sb_lock);
9196 }
9197
9198 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9199
9200 static const u16 sscdivintphase[] = {
9201         [BEND_IDX( 50)] = 0x3B23,
9202         [BEND_IDX( 45)] = 0x3B23,
9203         [BEND_IDX( 40)] = 0x3C23,
9204         [BEND_IDX( 35)] = 0x3C23,
9205         [BEND_IDX( 30)] = 0x3D23,
9206         [BEND_IDX( 25)] = 0x3D23,
9207         [BEND_IDX( 20)] = 0x3E23,
9208         [BEND_IDX( 15)] = 0x3E23,
9209         [BEND_IDX( 10)] = 0x3F23,
9210         [BEND_IDX(  5)] = 0x3F23,
9211         [BEND_IDX(  0)] = 0x0025,
9212         [BEND_IDX( -5)] = 0x0025,
9213         [BEND_IDX(-10)] = 0x0125,
9214         [BEND_IDX(-15)] = 0x0125,
9215         [BEND_IDX(-20)] = 0x0225,
9216         [BEND_IDX(-25)] = 0x0225,
9217         [BEND_IDX(-30)] = 0x0325,
9218         [BEND_IDX(-35)] = 0x0325,
9219         [BEND_IDX(-40)] = 0x0425,
9220         [BEND_IDX(-45)] = 0x0425,
9221         [BEND_IDX(-50)] = 0x0525,
9222 };
9223
9224 /*
9225  * Bend CLKOUT_DP
9226  * steps -50 to 50 inclusive, in steps of 5
9227  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9228  * change in clock period = -(steps / 10) * 5.787 ps
9229  */
9230 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9231 {
9232         u32 tmp;
9233         int idx = BEND_IDX(steps);
9234
9235         if (WARN_ON(steps % 5 != 0))
9236                 return;
9237
9238         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9239                 return;
9240
9241         mutex_lock(&dev_priv->sb_lock);
9242
9243         if (steps % 10 != 0)
9244                 tmp = 0xAAAAAAAB;
9245         else
9246                 tmp = 0x00000000;
9247         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9248
9249         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9250         tmp &= 0xffff0000;
9251         tmp |= sscdivintphase[idx];
9252         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9253
9254         mutex_unlock(&dev_priv->sb_lock);
9255 }
9256
9257 #undef BEND_IDX
9258
9259 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9260 {
9261         u32 fuse_strap = I915_READ(FUSE_STRAP);
9262         u32 ctl = I915_READ(SPLL_CTL);
9263
9264         if ((ctl & SPLL_PLL_ENABLE) == 0)
9265                 return false;
9266
9267         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9268             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9269                 return true;
9270
9271         if (IS_BROADWELL(dev_priv) &&
9272             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9273                 return true;
9274
9275         return false;
9276 }
9277
9278 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9279                                enum intel_dpll_id id)
9280 {
9281         u32 fuse_strap = I915_READ(FUSE_STRAP);
9282         u32 ctl = I915_READ(WRPLL_CTL(id));
9283
9284         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9285                 return false;
9286
9287         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9288                 return true;
9289
9290         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9291             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9292             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9293                 return true;
9294
9295         return false;
9296 }
9297
9298 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9299 {
9300         struct intel_encoder *encoder;
9301         bool pch_ssc_in_use = false;
9302         bool has_fdi = false;
9303
9304         for_each_intel_encoder(&dev_priv->drm, encoder) {
9305                 switch (encoder->type) {
9306                 case INTEL_OUTPUT_ANALOG:
9307                         has_fdi = true;
9308                         break;
9309                 default:
9310                         break;
9311                 }
9312         }
9313
9314         /*
9315          * The BIOS may have decided to use the PCH SSC
9316          * reference so we must not disable it until the
9317          * relevant PLLs have stopped relying on it. We'll
9318          * just leave the PCH SSC reference enabled in case
9319          * any active PLL is using it. It will get disabled
9320          * after runtime suspend if we don't have FDI.
9321          *
9322          * TODO: Move the whole reference clock handling
9323          * to the modeset sequence proper so that we can
9324          * actually enable/disable/reconfigure these things
9325          * safely. To do that we need to introduce a real
9326          * clock hierarchy. That would also allow us to do
9327          * clock bending finally.
9328          */
9329         if (spll_uses_pch_ssc(dev_priv)) {
9330                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9331                 pch_ssc_in_use = true;
9332         }
9333
9334         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9335                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9336                 pch_ssc_in_use = true;
9337         }
9338
9339         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9340                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9341                 pch_ssc_in_use = true;
9342         }
9343
9344         if (pch_ssc_in_use)
9345                 return;
9346
9347         if (has_fdi) {
9348                 lpt_bend_clkout_dp(dev_priv, 0);
9349                 lpt_enable_clkout_dp(dev_priv, true, true);
9350         } else {
9351                 lpt_disable_clkout_dp(dev_priv);
9352         }
9353 }
9354
9355 /*
9356  * Initialize reference clocks when the driver loads
9357  */
9358 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9359 {
9360         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9361                 ironlake_init_pch_refclk(dev_priv);
9362         else if (HAS_PCH_LPT(dev_priv))
9363                 lpt_init_pch_refclk(dev_priv);
9364 }
9365
9366 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9367 {
9368         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9369         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9370         enum pipe pipe = crtc->pipe;
9371         u32 val;
9372
9373         val = 0;
9374
9375         switch (crtc_state->pipe_bpp) {
9376         case 18:
9377                 val |= PIPECONF_6BPC;
9378                 break;
9379         case 24:
9380                 val |= PIPECONF_8BPC;
9381                 break;
9382         case 30:
9383                 val |= PIPECONF_10BPC;
9384                 break;
9385         case 36:
9386                 val |= PIPECONF_12BPC;
9387                 break;
9388         default:
9389                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9390                 BUG();
9391         }
9392
9393         if (crtc_state->dither)
9394                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9395
9396         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9397                 val |= PIPECONF_INTERLACED_ILK;
9398         else
9399                 val |= PIPECONF_PROGRESSIVE;
9400
9401         if (crtc_state->limited_color_range)
9402                 val |= PIPECONF_COLOR_RANGE_SELECT;
9403
9404         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9405
9406         I915_WRITE(PIPECONF(pipe), val);
9407         POSTING_READ(PIPECONF(pipe));
9408 }
9409
9410 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9411 {
9412         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9413         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9414         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9415         u32 val = 0;
9416
9417         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9418                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9419
9420         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9421                 val |= PIPECONF_INTERLACED_ILK;
9422         else
9423                 val |= PIPECONF_PROGRESSIVE;
9424
9425         I915_WRITE(PIPECONF(cpu_transcoder), val);
9426         POSTING_READ(PIPECONF(cpu_transcoder));
9427 }
9428
9429 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9430 {
9431         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9432         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9433         u32 val = 0;
9434
9435         switch (crtc_state->pipe_bpp) {
9436         case 18:
9437                 val |= PIPEMISC_DITHER_6_BPC;
9438                 break;
9439         case 24:
9440                 val |= PIPEMISC_DITHER_8_BPC;
9441                 break;
9442         case 30:
9443                 val |= PIPEMISC_DITHER_10_BPC;
9444                 break;
9445         case 36:
9446                 val |= PIPEMISC_DITHER_12_BPC;
9447                 break;
9448         default:
9449                 MISSING_CASE(crtc_state->pipe_bpp);
9450                 break;
9451         }
9452
9453         if (crtc_state->dither)
9454                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9455
9456         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9457             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9458                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9459
9460         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9461                 val |= PIPEMISC_YUV420_ENABLE |
9462                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9463
9464         if (INTEL_GEN(dev_priv) >= 11 &&
9465             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9466                                            BIT(PLANE_CURSOR))) == 0)
9467                 val |= PIPEMISC_HDR_MODE_PRECISION;
9468
9469         I915_WRITE(PIPEMISC(crtc->pipe), val);
9470 }
9471
9472 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9473 {
9474         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9475         u32 tmp;
9476
9477         tmp = I915_READ(PIPEMISC(crtc->pipe));
9478
9479         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9480         case PIPEMISC_DITHER_6_BPC:
9481                 return 18;
9482         case PIPEMISC_DITHER_8_BPC:
9483                 return 24;
9484         case PIPEMISC_DITHER_10_BPC:
9485                 return 30;
9486         case PIPEMISC_DITHER_12_BPC:
9487                 return 36;
9488         default:
9489                 MISSING_CASE(tmp);
9490                 return 0;
9491         }
9492 }
9493
9494 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9495 {
9496         /*
9497          * Account for spread spectrum to avoid
9498          * oversubscribing the link. Max center spread
9499          * is 2.5%; use 5% for safety's sake.
9500          */
9501         u32 bps = target_clock * bpp * 21 / 20;
9502         return DIV_ROUND_UP(bps, link_bw * 8);
9503 }
9504
9505 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9506 {
9507         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9508 }
9509
9510 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9511                                   struct intel_crtc_state *crtc_state,
9512                                   struct dpll *reduced_clock)
9513 {
9514         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9515         u32 dpll, fp, fp2;
9516         int factor;
9517
9518         /* Enable autotuning of the PLL clock (if permissible) */
9519         factor = 21;
9520         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9521                 if ((intel_panel_use_ssc(dev_priv) &&
9522                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9523                     (HAS_PCH_IBX(dev_priv) &&
9524                      intel_is_dual_link_lvds(dev_priv)))
9525                         factor = 25;
9526         } else if (crtc_state->sdvo_tv_clock) {
9527                 factor = 20;
9528         }
9529
9530         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9531
9532         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9533                 fp |= FP_CB_TUNE;
9534
9535         if (reduced_clock) {
9536                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9537
9538                 if (reduced_clock->m < factor * reduced_clock->n)
9539                         fp2 |= FP_CB_TUNE;
9540         } else {
9541                 fp2 = fp;
9542         }
9543
9544         dpll = 0;
9545
9546         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9547                 dpll |= DPLLB_MODE_LVDS;
9548         else
9549                 dpll |= DPLLB_MODE_DAC_SERIAL;
9550
9551         dpll |= (crtc_state->pixel_multiplier - 1)
9552                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9553
9554         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9555             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9556                 dpll |= DPLL_SDVO_HIGH_SPEED;
9557
9558         if (intel_crtc_has_dp_encoder(crtc_state))
9559                 dpll |= DPLL_SDVO_HIGH_SPEED;
9560
9561         /*
9562          * The high speed IO clock is only really required for
9563          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9564          * possible to share the DPLL between CRT and HDMI. Enabling
9565          * the clock needlessly does no real harm, except use up a
9566          * bit of power potentially.
9567          *
9568          * We'll limit this to IVB with 3 pipes, since it has only two
9569          * DPLLs and so DPLL sharing is the only way to get three pipes
9570          * driving PCH ports at the same time. On SNB we could do this,
9571          * and potentially avoid enabling the second DPLL, but it's not
9572          * clear if it''s a win or loss power wise. No point in doing
9573          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9574          */
9575         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
9576             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9577                 dpll |= DPLL_SDVO_HIGH_SPEED;
9578
9579         /* compute bitmask from p1 value */
9580         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9581         /* also FPA1 */
9582         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9583
9584         switch (crtc_state->dpll.p2) {
9585         case 5:
9586                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9587                 break;
9588         case 7:
9589                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9590                 break;
9591         case 10:
9592                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9593                 break;
9594         case 14:
9595                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9596                 break;
9597         }
9598
9599         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9600             intel_panel_use_ssc(dev_priv))
9601                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9602         else
9603                 dpll |= PLL_REF_INPUT_DREFCLK;
9604
9605         dpll |= DPLL_VCO_ENABLE;
9606
9607         crtc_state->dpll_hw_state.dpll = dpll;
9608         crtc_state->dpll_hw_state.fp0 = fp;
9609         crtc_state->dpll_hw_state.fp1 = fp2;
9610 }
9611
9612 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9613                                        struct intel_crtc_state *crtc_state)
9614 {
9615         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9616         struct intel_atomic_state *state =
9617                 to_intel_atomic_state(crtc_state->base.state);
9618         const struct intel_limit *limit;
9619         int refclk = 120000;
9620
9621         memset(&crtc_state->dpll_hw_state, 0,
9622                sizeof(crtc_state->dpll_hw_state));
9623
9624         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9625         if (!crtc_state->has_pch_encoder)
9626                 return 0;
9627
9628         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9629                 if (intel_panel_use_ssc(dev_priv)) {
9630                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9631                                       dev_priv->vbt.lvds_ssc_freq);
9632                         refclk = dev_priv->vbt.lvds_ssc_freq;
9633                 }
9634
9635                 if (intel_is_dual_link_lvds(dev_priv)) {
9636                         if (refclk == 100000)
9637                                 limit = &intel_limits_ironlake_dual_lvds_100m;
9638                         else
9639                                 limit = &intel_limits_ironlake_dual_lvds;
9640                 } else {
9641                         if (refclk == 100000)
9642                                 limit = &intel_limits_ironlake_single_lvds_100m;
9643                         else
9644                                 limit = &intel_limits_ironlake_single_lvds;
9645                 }
9646         } else {
9647                 limit = &intel_limits_ironlake_dac;
9648         }
9649
9650         if (!crtc_state->clock_set &&
9651             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9652                                 refclk, NULL, &crtc_state->dpll)) {
9653                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
9654                 return -EINVAL;
9655         }
9656
9657         ironlake_compute_dpll(crtc, crtc_state, NULL);
9658
9659         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
9660                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9661                               pipe_name(crtc->pipe));
9662                 return -EINVAL;
9663         }
9664
9665         return 0;
9666 }
9667
9668 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
9669                                          struct intel_link_m_n *m_n)
9670 {
9671         struct drm_device *dev = crtc->base.dev;
9672         struct drm_i915_private *dev_priv = to_i915(dev);
9673         enum pipe pipe = crtc->pipe;
9674
9675         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9676         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9677         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9678                 & ~TU_SIZE_MASK;
9679         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9680         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9681                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9682 }
9683
9684 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9685                                          enum transcoder transcoder,
9686                                          struct intel_link_m_n *m_n,
9687                                          struct intel_link_m_n *m2_n2)
9688 {
9689         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9690         enum pipe pipe = crtc->pipe;
9691
9692         if (INTEL_GEN(dev_priv) >= 5) {
9693                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9694                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9695                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9696                         & ~TU_SIZE_MASK;
9697                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9698                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9699                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9700
9701                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
9702                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9703                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9704                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9705                                         & ~TU_SIZE_MASK;
9706                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9707                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9708                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9709                 }
9710         } else {
9711                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9712                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9713                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9714                         & ~TU_SIZE_MASK;
9715                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9716                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9717                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9718         }
9719 }
9720
9721 void intel_dp_get_m_n(struct intel_crtc *crtc,
9722                       struct intel_crtc_state *pipe_config)
9723 {
9724         if (pipe_config->has_pch_encoder)
9725                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9726         else
9727                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9728                                              &pipe_config->dp_m_n,
9729                                              &pipe_config->dp_m2_n2);
9730 }
9731
9732 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9733                                         struct intel_crtc_state *pipe_config)
9734 {
9735         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9736                                      &pipe_config->fdi_m_n, NULL);
9737 }
9738
9739 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9740                                     struct intel_crtc_state *pipe_config)
9741 {
9742         struct drm_device *dev = crtc->base.dev;
9743         struct drm_i915_private *dev_priv = to_i915(dev);
9744         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9745         u32 ps_ctrl = 0;
9746         int id = -1;
9747         int i;
9748
9749         /* find scaler attached to this pipe */
9750         for (i = 0; i < crtc->num_scalers; i++) {
9751                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9752                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9753                         id = i;
9754                         pipe_config->pch_pfit.enabled = true;
9755                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9756                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9757                         scaler_state->scalers[i].in_use = true;
9758                         break;
9759                 }
9760         }
9761
9762         scaler_state->scaler_id = id;
9763         if (id >= 0) {
9764                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9765         } else {
9766                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9767         }
9768 }
9769
9770 static void
9771 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9772                                  struct intel_initial_plane_config *plane_config)
9773 {
9774         struct drm_device *dev = crtc->base.dev;
9775         struct drm_i915_private *dev_priv = to_i915(dev);
9776         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9777         enum plane_id plane_id = plane->id;
9778         enum pipe pipe;
9779         u32 val, base, offset, stride_mult, tiling, alpha;
9780         int fourcc, pixel_format;
9781         unsigned int aligned_height;
9782         struct drm_framebuffer *fb;
9783         struct intel_framebuffer *intel_fb;
9784
9785         if (!plane->get_hw_state(plane, &pipe))
9786                 return;
9787
9788         WARN_ON(pipe != crtc->pipe);
9789
9790         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9791         if (!intel_fb) {
9792                 DRM_DEBUG_KMS("failed to alloc fb\n");
9793                 return;
9794         }
9795
9796         fb = &intel_fb->base;
9797
9798         fb->dev = dev;
9799
9800         val = I915_READ(PLANE_CTL(pipe, plane_id));
9801
9802         if (INTEL_GEN(dev_priv) >= 11)
9803                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
9804         else
9805                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
9806
9807         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
9808                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
9809                 alpha &= PLANE_COLOR_ALPHA_MASK;
9810         } else {
9811                 alpha = val & PLANE_CTL_ALPHA_MASK;
9812         }
9813
9814         fourcc = skl_format_to_fourcc(pixel_format,
9815                                       val & PLANE_CTL_ORDER_RGBX, alpha);
9816         fb->format = drm_format_info(fourcc);
9817
9818         tiling = val & PLANE_CTL_TILED_MASK;
9819         switch (tiling) {
9820         case PLANE_CTL_TILED_LINEAR:
9821                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
9822                 break;
9823         case PLANE_CTL_TILED_X:
9824                 plane_config->tiling = I915_TILING_X;
9825                 fb->modifier = I915_FORMAT_MOD_X_TILED;
9826                 break;
9827         case PLANE_CTL_TILED_Y:
9828                 plane_config->tiling = I915_TILING_Y;
9829                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9830                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
9831                 else
9832                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
9833                 break;
9834         case PLANE_CTL_TILED_YF:
9835                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9836                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
9837                 else
9838                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
9839                 break;
9840         default:
9841                 MISSING_CASE(tiling);
9842                 goto error;
9843         }
9844
9845         /*
9846          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
9847          * while i915 HW rotation is clockwise, thats why this swapping.
9848          */
9849         switch (val & PLANE_CTL_ROTATE_MASK) {
9850         case PLANE_CTL_ROTATE_0:
9851                 plane_config->rotation = DRM_MODE_ROTATE_0;
9852                 break;
9853         case PLANE_CTL_ROTATE_90:
9854                 plane_config->rotation = DRM_MODE_ROTATE_270;
9855                 break;
9856         case PLANE_CTL_ROTATE_180:
9857                 plane_config->rotation = DRM_MODE_ROTATE_180;
9858                 break;
9859         case PLANE_CTL_ROTATE_270:
9860                 plane_config->rotation = DRM_MODE_ROTATE_90;
9861                 break;
9862         }
9863
9864         if (INTEL_GEN(dev_priv) >= 10 &&
9865             val & PLANE_CTL_FLIP_HORIZONTAL)
9866                 plane_config->rotation |= DRM_MODE_REFLECT_X;
9867
9868         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
9869         plane_config->base = base;
9870
9871         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
9872
9873         val = I915_READ(PLANE_SIZE(pipe, plane_id));
9874         fb->height = ((val >> 16) & 0xfff) + 1;
9875         fb->width = ((val >> 0) & 0x1fff) + 1;
9876
9877         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
9878         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
9879         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9880
9881         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9882
9883         plane_config->size = fb->pitches[0] * aligned_height;
9884
9885         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9886                       crtc->base.name, plane->base.name, fb->width, fb->height,
9887                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9888                       plane_config->size);
9889
9890         plane_config->fb = intel_fb;
9891         return;
9892
9893 error:
9894         kfree(intel_fb);
9895 }
9896
9897 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9898                                      struct intel_crtc_state *pipe_config)
9899 {
9900         struct drm_device *dev = crtc->base.dev;
9901         struct drm_i915_private *dev_priv = to_i915(dev);
9902         u32 tmp;
9903
9904         tmp = I915_READ(PF_CTL(crtc->pipe));
9905
9906         if (tmp & PF_ENABLE) {
9907                 pipe_config->pch_pfit.enabled = true;
9908                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9909                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9910
9911                 /* We currently do not free assignements of panel fitters on
9912                  * ivb/hsw (since we don't use the higher upscaling modes which
9913                  * differentiates them) so just WARN about this case for now. */
9914                 if (IS_GEN(dev_priv, 7)) {
9915                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9916                                 PF_PIPE_SEL_IVB(crtc->pipe));
9917                 }
9918         }
9919 }
9920
9921 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9922                                      struct intel_crtc_state *pipe_config)
9923 {
9924         struct drm_device *dev = crtc->base.dev;
9925         struct drm_i915_private *dev_priv = to_i915(dev);
9926         enum intel_display_power_domain power_domain;
9927         intel_wakeref_t wakeref;
9928         u32 tmp;
9929         bool ret;
9930
9931         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9932         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9933         if (!wakeref)
9934                 return false;
9935
9936         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9937         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9938         pipe_config->shared_dpll = NULL;
9939
9940         ret = false;
9941         tmp = I915_READ(PIPECONF(crtc->pipe));
9942         if (!(tmp & PIPECONF_ENABLE))
9943                 goto out;
9944
9945         switch (tmp & PIPECONF_BPC_MASK) {
9946         case PIPECONF_6BPC:
9947                 pipe_config->pipe_bpp = 18;
9948                 break;
9949         case PIPECONF_8BPC:
9950                 pipe_config->pipe_bpp = 24;
9951                 break;
9952         case PIPECONF_10BPC:
9953                 pipe_config->pipe_bpp = 30;
9954                 break;
9955         case PIPECONF_12BPC:
9956                 pipe_config->pipe_bpp = 36;
9957                 break;
9958         default:
9959                 break;
9960         }
9961
9962         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9963                 pipe_config->limited_color_range = true;
9964
9965         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
9966                 PIPECONF_GAMMA_MODE_SHIFT;
9967
9968         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
9969
9970         i9xx_get_pipe_color_config(pipe_config);
9971         intel_color_get_config(pipe_config);
9972
9973         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9974                 struct intel_shared_dpll *pll;
9975                 enum intel_dpll_id pll_id;
9976
9977                 pipe_config->has_pch_encoder = true;
9978
9979                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9980                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9981                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9982
9983                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9984
9985                 if (HAS_PCH_IBX(dev_priv)) {
9986                         /*
9987                          * The pipe->pch transcoder and pch transcoder->pll
9988                          * mapping is fixed.
9989                          */
9990                         pll_id = (enum intel_dpll_id) crtc->pipe;
9991                 } else {
9992                         tmp = I915_READ(PCH_DPLL_SEL);
9993                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9994                                 pll_id = DPLL_ID_PCH_PLL_B;
9995                         else
9996                                 pll_id= DPLL_ID_PCH_PLL_A;
9997                 }
9998
9999                 pipe_config->shared_dpll =
10000                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10001                 pll = pipe_config->shared_dpll;
10002
10003                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10004                                                 &pipe_config->dpll_hw_state));
10005
10006                 tmp = pipe_config->dpll_hw_state.dpll;
10007                 pipe_config->pixel_multiplier =
10008                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10009                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10010
10011                 ironlake_pch_clock_get(crtc, pipe_config);
10012         } else {
10013                 pipe_config->pixel_multiplier = 1;
10014         }
10015
10016         intel_get_pipe_timings(crtc, pipe_config);
10017         intel_get_pipe_src_size(crtc, pipe_config);
10018
10019         ironlake_get_pfit_config(crtc, pipe_config);
10020
10021         ret = true;
10022
10023 out:
10024         intel_display_power_put(dev_priv, power_domain, wakeref);
10025
10026         return ret;
10027 }
10028 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10029                                       struct intel_crtc_state *crtc_state)
10030 {
10031         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10032         struct intel_atomic_state *state =
10033                 to_intel_atomic_state(crtc_state->base.state);
10034
10035         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10036             INTEL_GEN(dev_priv) >= 11) {
10037                 struct intel_encoder *encoder =
10038                         intel_get_crtc_new_encoder(state, crtc_state);
10039
10040                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10041                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10042                                       pipe_name(crtc->pipe));
10043                         return -EINVAL;
10044                 }
10045         }
10046
10047         return 0;
10048 }
10049
10050 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10051                                    enum port port,
10052                                    struct intel_crtc_state *pipe_config)
10053 {
10054         enum intel_dpll_id id;
10055         u32 temp;
10056
10057         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10058         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10059
10060         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10061                 return;
10062
10063         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10064 }
10065
10066 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10067                                 enum port port,
10068                                 struct intel_crtc_state *pipe_config)
10069 {
10070         enum phy phy = intel_port_to_phy(dev_priv, port);
10071         enum icl_port_dpll_id port_dpll_id;
10072         enum intel_dpll_id id;
10073         u32 temp;
10074
10075         if (intel_phy_is_combo(dev_priv, phy)) {
10076                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10077                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10078                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10079                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10080         } else if (intel_phy_is_tc(dev_priv, phy)) {
10081                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10082
10083                 if (clk_sel == DDI_CLK_SEL_MG) {
10084                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10085                                                                     port));
10086                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10087                 } else {
10088                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10089                         id = DPLL_ID_ICL_TBTPLL;
10090                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10091                 }
10092         } else {
10093                 WARN(1, "Invalid port %x\n", port);
10094                 return;
10095         }
10096
10097         pipe_config->icl_port_dplls[port_dpll_id].pll =
10098                 intel_get_shared_dpll_by_id(dev_priv, id);
10099
10100         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10101 }
10102
10103 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10104                                 enum port port,
10105                                 struct intel_crtc_state *pipe_config)
10106 {
10107         enum intel_dpll_id id;
10108
10109         switch (port) {
10110         case PORT_A:
10111                 id = DPLL_ID_SKL_DPLL0;
10112                 break;
10113         case PORT_B:
10114                 id = DPLL_ID_SKL_DPLL1;
10115                 break;
10116         case PORT_C:
10117                 id = DPLL_ID_SKL_DPLL2;
10118                 break;
10119         default:
10120                 DRM_ERROR("Incorrect port type\n");
10121                 return;
10122         }
10123
10124         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10125 }
10126
10127 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10128                                 enum port port,
10129                                 struct intel_crtc_state *pipe_config)
10130 {
10131         enum intel_dpll_id id;
10132         u32 temp;
10133
10134         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10135         id = temp >> (port * 3 + 1);
10136
10137         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10138                 return;
10139
10140         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10141 }
10142
10143 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10144                                 enum port port,
10145                                 struct intel_crtc_state *pipe_config)
10146 {
10147         enum intel_dpll_id id;
10148         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10149
10150         switch (ddi_pll_sel) {
10151         case PORT_CLK_SEL_WRPLL1:
10152                 id = DPLL_ID_WRPLL1;
10153                 break;
10154         case PORT_CLK_SEL_WRPLL2:
10155                 id = DPLL_ID_WRPLL2;
10156                 break;
10157         case PORT_CLK_SEL_SPLL:
10158                 id = DPLL_ID_SPLL;
10159                 break;
10160         case PORT_CLK_SEL_LCPLL_810:
10161                 id = DPLL_ID_LCPLL_810;
10162                 break;
10163         case PORT_CLK_SEL_LCPLL_1350:
10164                 id = DPLL_ID_LCPLL_1350;
10165                 break;
10166         case PORT_CLK_SEL_LCPLL_2700:
10167                 id = DPLL_ID_LCPLL_2700;
10168                 break;
10169         default:
10170                 MISSING_CASE(ddi_pll_sel);
10171                 /* fall through */
10172         case PORT_CLK_SEL_NONE:
10173                 return;
10174         }
10175
10176         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10177 }
10178
10179 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10180                                      struct intel_crtc_state *pipe_config,
10181                                      u64 *power_domain_mask,
10182                                      intel_wakeref_t *wakerefs)
10183 {
10184         struct drm_device *dev = crtc->base.dev;
10185         struct drm_i915_private *dev_priv = to_i915(dev);
10186         enum intel_display_power_domain power_domain;
10187         unsigned long panel_transcoder_mask = 0;
10188         unsigned long enabled_panel_transcoders = 0;
10189         enum transcoder panel_transcoder;
10190         intel_wakeref_t wf;
10191         u32 tmp;
10192
10193         if (INTEL_GEN(dev_priv) >= 11)
10194                 panel_transcoder_mask |=
10195                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10196
10197         if (HAS_TRANSCODER_EDP(dev_priv))
10198                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10199
10200         /*
10201          * The pipe->transcoder mapping is fixed with the exception of the eDP
10202          * and DSI transcoders handled below.
10203          */
10204         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10205
10206         /*
10207          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10208          * consistency and less surprising code; it's in always on power).
10209          */
10210         for_each_set_bit(panel_transcoder,
10211                          &panel_transcoder_mask,
10212                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10213                 bool force_thru = false;
10214                 enum pipe trans_pipe;
10215
10216                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10217                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10218                         continue;
10219
10220                 /*
10221                  * Log all enabled ones, only use the first one.
10222                  *
10223                  * FIXME: This won't work for two separate DSI displays.
10224                  */
10225                 enabled_panel_transcoders |= BIT(panel_transcoder);
10226                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10227                         continue;
10228
10229                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10230                 default:
10231                         WARN(1, "unknown pipe linked to transcoder %s\n",
10232                              transcoder_name(panel_transcoder));
10233                         /* fall through */
10234                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10235                         force_thru = true;
10236                         /* fall through */
10237                 case TRANS_DDI_EDP_INPUT_A_ON:
10238                         trans_pipe = PIPE_A;
10239                         break;
10240                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10241                         trans_pipe = PIPE_B;
10242                         break;
10243                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10244                         trans_pipe = PIPE_C;
10245                         break;
10246                 }
10247
10248                 if (trans_pipe == crtc->pipe) {
10249                         pipe_config->cpu_transcoder = panel_transcoder;
10250                         pipe_config->pch_pfit.force_thru = force_thru;
10251                 }
10252         }
10253
10254         /*
10255          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10256          */
10257         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10258                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10259
10260         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10261         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10262
10263         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10264         if (!wf)
10265                 return false;
10266
10267         wakerefs[power_domain] = wf;
10268         *power_domain_mask |= BIT_ULL(power_domain);
10269
10270         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10271
10272         return tmp & PIPECONF_ENABLE;
10273 }
10274
10275 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10276                                          struct intel_crtc_state *pipe_config,
10277                                          u64 *power_domain_mask,
10278                                          intel_wakeref_t *wakerefs)
10279 {
10280         struct drm_device *dev = crtc->base.dev;
10281         struct drm_i915_private *dev_priv = to_i915(dev);
10282         enum intel_display_power_domain power_domain;
10283         enum transcoder cpu_transcoder;
10284         intel_wakeref_t wf;
10285         enum port port;
10286         u32 tmp;
10287
10288         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10289                 if (port == PORT_A)
10290                         cpu_transcoder = TRANSCODER_DSI_A;
10291                 else
10292                         cpu_transcoder = TRANSCODER_DSI_C;
10293
10294                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10295                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10296
10297                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10298                 if (!wf)
10299                         continue;
10300
10301                 wakerefs[power_domain] = wf;
10302                 *power_domain_mask |= BIT_ULL(power_domain);
10303
10304                 /*
10305                  * The PLL needs to be enabled with a valid divider
10306                  * configuration, otherwise accessing DSI registers will hang
10307                  * the machine. See BSpec North Display Engine
10308                  * registers/MIPI[BXT]. We can break out here early, since we
10309                  * need the same DSI PLL to be enabled for both DSI ports.
10310                  */
10311                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10312                         break;
10313
10314                 /* XXX: this works for video mode only */
10315                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10316                 if (!(tmp & DPI_ENABLE))
10317                         continue;
10318
10319                 tmp = I915_READ(MIPI_CTRL(port));
10320                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10321                         continue;
10322
10323                 pipe_config->cpu_transcoder = cpu_transcoder;
10324                 break;
10325         }
10326
10327         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10328 }
10329
10330 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10331                                        struct intel_crtc_state *pipe_config)
10332 {
10333         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10334         struct intel_shared_dpll *pll;
10335         enum port port;
10336         u32 tmp;
10337
10338         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
10339
10340         if (INTEL_GEN(dev_priv) >= 12)
10341                 port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10342         else
10343                 port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10344
10345         if (INTEL_GEN(dev_priv) >= 11)
10346                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10347         else if (IS_CANNONLAKE(dev_priv))
10348                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10349         else if (IS_GEN9_BC(dev_priv))
10350                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10351         else if (IS_GEN9_LP(dev_priv))
10352                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10353         else
10354                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10355
10356         pll = pipe_config->shared_dpll;
10357         if (pll) {
10358                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10359                                                 &pipe_config->dpll_hw_state));
10360         }
10361
10362         /*
10363          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10364          * DDI E. So just check whether this pipe is wired to DDI E and whether
10365          * the PCH transcoder is on.
10366          */
10367         if (INTEL_GEN(dev_priv) < 9 &&
10368             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10369                 pipe_config->has_pch_encoder = true;
10370
10371                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10372                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10373                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10374
10375                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10376         }
10377 }
10378
10379 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10380                                     struct intel_crtc_state *pipe_config)
10381 {
10382         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10383         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10384         enum intel_display_power_domain power_domain;
10385         u64 power_domain_mask;
10386         bool active;
10387
10388         intel_crtc_init_scalers(crtc, pipe_config);
10389
10390         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10391         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10392         if (!wf)
10393                 return false;
10394
10395         wakerefs[power_domain] = wf;
10396         power_domain_mask = BIT_ULL(power_domain);
10397
10398         pipe_config->shared_dpll = NULL;
10399
10400         active = hsw_get_transcoder_state(crtc, pipe_config,
10401                                           &power_domain_mask, wakerefs);
10402
10403         if (IS_GEN9_LP(dev_priv) &&
10404             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10405                                          &power_domain_mask, wakerefs)) {
10406                 WARN_ON(active);
10407                 active = true;
10408         }
10409
10410         if (!active)
10411                 goto out;
10412
10413         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10414             INTEL_GEN(dev_priv) >= 11) {
10415                 haswell_get_ddi_port_state(crtc, pipe_config);
10416                 intel_get_pipe_timings(crtc, pipe_config);
10417         }
10418
10419         intel_get_pipe_src_size(crtc, pipe_config);
10420         intel_get_crtc_ycbcr_config(crtc, pipe_config);
10421
10422         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10423
10424         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10425
10426         if (INTEL_GEN(dev_priv) >= 9) {
10427                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10428
10429                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10430                         pipe_config->gamma_enable = true;
10431
10432                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10433                         pipe_config->csc_enable = true;
10434         } else {
10435                 i9xx_get_pipe_color_config(pipe_config);
10436         }
10437
10438         intel_color_get_config(pipe_config);
10439
10440         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10441         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10442
10443         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10444         if (wf) {
10445                 wakerefs[power_domain] = wf;
10446                 power_domain_mask |= BIT_ULL(power_domain);
10447
10448                 if (INTEL_GEN(dev_priv) >= 9)
10449                         skylake_get_pfit_config(crtc, pipe_config);
10450                 else
10451                         ironlake_get_pfit_config(crtc, pipe_config);
10452         }
10453
10454         if (hsw_crtc_supports_ips(crtc)) {
10455                 if (IS_HASWELL(dev_priv))
10456                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10457                 else {
10458                         /*
10459                          * We cannot readout IPS state on broadwell, set to
10460                          * true so we can set it to a defined state on first
10461                          * commit.
10462                          */
10463                         pipe_config->ips_enabled = true;
10464                 }
10465         }
10466
10467         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10468             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10469                 pipe_config->pixel_multiplier =
10470                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10471         } else {
10472                 pipe_config->pixel_multiplier = 1;
10473         }
10474
10475 out:
10476         for_each_power_domain(power_domain, power_domain_mask)
10477                 intel_display_power_put(dev_priv,
10478                                         power_domain, wakerefs[power_domain]);
10479
10480         return active;
10481 }
10482
10483 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10484 {
10485         struct drm_i915_private *dev_priv =
10486                 to_i915(plane_state->base.plane->dev);
10487         const struct drm_framebuffer *fb = plane_state->base.fb;
10488         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10489         u32 base;
10490
10491         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10492                 base = obj->phys_handle->busaddr;
10493         else
10494                 base = intel_plane_ggtt_offset(plane_state);
10495
10496         base += plane_state->color_plane[0].offset;
10497
10498         /* ILK+ do this automagically */
10499         if (HAS_GMCH(dev_priv) &&
10500             plane_state->base.rotation & DRM_MODE_ROTATE_180)
10501                 base += (plane_state->base.crtc_h *
10502                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
10503
10504         return base;
10505 }
10506
10507 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10508 {
10509         int x = plane_state->base.crtc_x;
10510         int y = plane_state->base.crtc_y;
10511         u32 pos = 0;
10512
10513         if (x < 0) {
10514                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10515                 x = -x;
10516         }
10517         pos |= x << CURSOR_X_SHIFT;
10518
10519         if (y < 0) {
10520                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10521                 y = -y;
10522         }
10523         pos |= y << CURSOR_Y_SHIFT;
10524
10525         return pos;
10526 }
10527
10528 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10529 {
10530         const struct drm_mode_config *config =
10531                 &plane_state->base.plane->dev->mode_config;
10532         int width = plane_state->base.crtc_w;
10533         int height = plane_state->base.crtc_h;
10534
10535         return width > 0 && width <= config->cursor_width &&
10536                 height > 0 && height <= config->cursor_height;
10537 }
10538
10539 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10540 {
10541         int src_x, src_y;
10542         u32 offset;
10543         int ret;
10544
10545         ret = intel_plane_compute_gtt(plane_state);
10546         if (ret)
10547                 return ret;
10548
10549         if (!plane_state->base.visible)
10550                 return 0;
10551
10552         src_x = plane_state->base.src_x >> 16;
10553         src_y = plane_state->base.src_y >> 16;
10554
10555         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
10556         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
10557                                                     plane_state, 0);
10558
10559         if (src_x != 0 || src_y != 0) {
10560                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
10561                 return -EINVAL;
10562         }
10563
10564         plane_state->color_plane[0].offset = offset;
10565
10566         return 0;
10567 }
10568
10569 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
10570                               struct intel_plane_state *plane_state)
10571 {
10572         const struct drm_framebuffer *fb = plane_state->base.fb;
10573         int ret;
10574
10575         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
10576                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
10577                 return -EINVAL;
10578         }
10579
10580         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
10581                                                   &crtc_state->base,
10582                                                   DRM_PLANE_HELPER_NO_SCALING,
10583                                                   DRM_PLANE_HELPER_NO_SCALING,
10584                                                   true, true);
10585         if (ret)
10586                 return ret;
10587
10588         ret = intel_cursor_check_surface(plane_state);
10589         if (ret)
10590                 return ret;
10591
10592         if (!plane_state->base.visible)
10593                 return 0;
10594
10595         ret = intel_plane_check_src_coordinates(plane_state);
10596         if (ret)
10597                 return ret;
10598
10599         return 0;
10600 }
10601
10602 static unsigned int
10603 i845_cursor_max_stride(struct intel_plane *plane,
10604                        u32 pixel_format, u64 modifier,
10605                        unsigned int rotation)
10606 {
10607         return 2048;
10608 }
10609
10610 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10611 {
10612         u32 cntl = 0;
10613
10614         if (crtc_state->gamma_enable)
10615                 cntl |= CURSOR_GAMMA_ENABLE;
10616
10617         return cntl;
10618 }
10619
10620 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
10621                            const struct intel_plane_state *plane_state)
10622 {
10623         return CURSOR_ENABLE |
10624                 CURSOR_FORMAT_ARGB |
10625                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
10626 }
10627
10628 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
10629 {
10630         int width = plane_state->base.crtc_w;
10631
10632         /*
10633          * 845g/865g are only limited by the width of their cursors,
10634          * the height is arbitrary up to the precision of the register.
10635          */
10636         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
10637 }
10638
10639 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
10640                              struct intel_plane_state *plane_state)
10641 {
10642         const struct drm_framebuffer *fb = plane_state->base.fb;
10643         int ret;
10644
10645         ret = intel_check_cursor(crtc_state, plane_state);
10646         if (ret)
10647                 return ret;
10648
10649         /* if we want to turn off the cursor ignore width and height */
10650         if (!fb)
10651                 return 0;
10652
10653         /* Check for which cursor types we support */
10654         if (!i845_cursor_size_ok(plane_state)) {
10655                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10656                           plane_state->base.crtc_w,
10657                           plane_state->base.crtc_h);
10658                 return -EINVAL;
10659         }
10660
10661         WARN_ON(plane_state->base.visible &&
10662                 plane_state->color_plane[0].stride != fb->pitches[0]);
10663
10664         switch (fb->pitches[0]) {
10665         case 256:
10666         case 512:
10667         case 1024:
10668         case 2048:
10669                 break;
10670         default:
10671                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
10672                               fb->pitches[0]);
10673                 return -EINVAL;
10674         }
10675
10676         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
10677
10678         return 0;
10679 }
10680
10681 static void i845_update_cursor(struct intel_plane *plane,
10682                                const struct intel_crtc_state *crtc_state,
10683                                const struct intel_plane_state *plane_state)
10684 {
10685         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10686         u32 cntl = 0, base = 0, pos = 0, size = 0;
10687         unsigned long irqflags;
10688
10689         if (plane_state && plane_state->base.visible) {
10690                 unsigned int width = plane_state->base.crtc_w;
10691                 unsigned int height = plane_state->base.crtc_h;
10692
10693                 cntl = plane_state->ctl |
10694                         i845_cursor_ctl_crtc(crtc_state);
10695
10696                 size = (height << 12) | width;
10697
10698                 base = intel_cursor_base(plane_state);
10699                 pos = intel_cursor_position(plane_state);
10700         }
10701
10702         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10703
10704         /* On these chipsets we can only modify the base/size/stride
10705          * whilst the cursor is disabled.
10706          */
10707         if (plane->cursor.base != base ||
10708             plane->cursor.size != size ||
10709             plane->cursor.cntl != cntl) {
10710                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
10711                 I915_WRITE_FW(CURBASE(PIPE_A), base);
10712                 I915_WRITE_FW(CURSIZE, size);
10713                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10714                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
10715
10716                 plane->cursor.base = base;
10717                 plane->cursor.size = size;
10718                 plane->cursor.cntl = cntl;
10719         } else {
10720                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10721         }
10722
10723         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10724 }
10725
10726 static void i845_disable_cursor(struct intel_plane *plane,
10727                                 const struct intel_crtc_state *crtc_state)
10728 {
10729         i845_update_cursor(plane, crtc_state, NULL);
10730 }
10731
10732 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
10733                                      enum pipe *pipe)
10734 {
10735         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10736         enum intel_display_power_domain power_domain;
10737         intel_wakeref_t wakeref;
10738         bool ret;
10739
10740         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
10741         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10742         if (!wakeref)
10743                 return false;
10744
10745         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
10746
10747         *pipe = PIPE_A;
10748
10749         intel_display_power_put(dev_priv, power_domain, wakeref);
10750
10751         return ret;
10752 }
10753
10754 static unsigned int
10755 i9xx_cursor_max_stride(struct intel_plane *plane,
10756                        u32 pixel_format, u64 modifier,
10757                        unsigned int rotation)
10758 {
10759         return plane->base.dev->mode_config.cursor_width * 4;
10760 }
10761
10762 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10763 {
10764         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10765         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10766         u32 cntl = 0;
10767
10768         if (INTEL_GEN(dev_priv) >= 11)
10769                 return cntl;
10770
10771         if (crtc_state->gamma_enable)
10772                 cntl = MCURSOR_GAMMA_ENABLE;
10773
10774         if (crtc_state->csc_enable)
10775                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
10776
10777         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10778                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10779
10780         return cntl;
10781 }
10782
10783 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
10784                            const struct intel_plane_state *plane_state)
10785 {
10786         struct drm_i915_private *dev_priv =
10787                 to_i915(plane_state->base.plane->dev);
10788         u32 cntl = 0;
10789
10790         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
10791                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10792
10793         switch (plane_state->base.crtc_w) {
10794         case 64:
10795                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10796                 break;
10797         case 128:
10798                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10799                 break;
10800         case 256:
10801                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10802                 break;
10803         default:
10804                 MISSING_CASE(plane_state->base.crtc_w);
10805                 return 0;
10806         }
10807
10808         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10809                 cntl |= MCURSOR_ROTATE_180;
10810
10811         return cntl;
10812 }
10813
10814 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10815 {
10816         struct drm_i915_private *dev_priv =
10817                 to_i915(plane_state->base.plane->dev);
10818         int width = plane_state->base.crtc_w;
10819         int height = plane_state->base.crtc_h;
10820
10821         if (!intel_cursor_size_ok(plane_state))
10822                 return false;
10823
10824         /* Cursor width is limited to a few power-of-two sizes */
10825         switch (width) {
10826         case 256:
10827         case 128:
10828         case 64:
10829                 break;
10830         default:
10831                 return false;
10832         }
10833
10834         /*
10835          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10836          * height from 8 lines up to the cursor width, when the
10837          * cursor is not rotated. Everything else requires square
10838          * cursors.
10839          */
10840         if (HAS_CUR_FBC(dev_priv) &&
10841             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10842                 if (height < 8 || height > width)
10843                         return false;
10844         } else {
10845                 if (height != width)
10846                         return false;
10847         }
10848
10849         return true;
10850 }
10851
10852 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10853                              struct intel_plane_state *plane_state)
10854 {
10855         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10856         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10857         const struct drm_framebuffer *fb = plane_state->base.fb;
10858         enum pipe pipe = plane->pipe;
10859         int ret;
10860
10861         ret = intel_check_cursor(crtc_state, plane_state);
10862         if (ret)
10863                 return ret;
10864
10865         /* if we want to turn off the cursor ignore width and height */
10866         if (!fb)
10867                 return 0;
10868
10869         /* Check for which cursor types we support */
10870         if (!i9xx_cursor_size_ok(plane_state)) {
10871                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10872                           plane_state->base.crtc_w,
10873                           plane_state->base.crtc_h);
10874                 return -EINVAL;
10875         }
10876
10877         WARN_ON(plane_state->base.visible &&
10878                 plane_state->color_plane[0].stride != fb->pitches[0]);
10879
10880         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10881                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10882                               fb->pitches[0], plane_state->base.crtc_w);
10883                 return -EINVAL;
10884         }
10885
10886         /*
10887          * There's something wrong with the cursor on CHV pipe C.
10888          * If it straddles the left edge of the screen then
10889          * moving it away from the edge or disabling it often
10890          * results in a pipe underrun, and often that can lead to
10891          * dead pipe (constant underrun reported, and it scans
10892          * out just a solid color). To recover from that, the
10893          * display power well must be turned off and on again.
10894          * Refuse the put the cursor into that compromised position.
10895          */
10896         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10897             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10898                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10899                 return -EINVAL;
10900         }
10901
10902         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10903
10904         return 0;
10905 }
10906
10907 static void i9xx_update_cursor(struct intel_plane *plane,
10908                                const struct intel_crtc_state *crtc_state,
10909                                const struct intel_plane_state *plane_state)
10910 {
10911         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10912         enum pipe pipe = plane->pipe;
10913         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10914         unsigned long irqflags;
10915
10916         if (plane_state && plane_state->base.visible) {
10917                 cntl = plane_state->ctl |
10918                         i9xx_cursor_ctl_crtc(crtc_state);
10919
10920                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10921                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10922
10923                 base = intel_cursor_base(plane_state);
10924                 pos = intel_cursor_position(plane_state);
10925         }
10926
10927         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10928
10929         /*
10930          * On some platforms writing CURCNTR first will also
10931          * cause CURPOS to be armed by the CURBASE write.
10932          * Without the CURCNTR write the CURPOS write would
10933          * arm itself. Thus we always update CURCNTR before
10934          * CURPOS.
10935          *
10936          * On other platforms CURPOS always requires the
10937          * CURBASE write to arm the update. Additonally
10938          * a write to any of the cursor register will cancel
10939          * an already armed cursor update. Thus leaving out
10940          * the CURBASE write after CURPOS could lead to a
10941          * cursor that doesn't appear to move, or even change
10942          * shape. Thus we always write CURBASE.
10943          *
10944          * The other registers are armed by by the CURBASE write
10945          * except when the plane is getting enabled at which time
10946          * the CURCNTR write arms the update.
10947          */
10948
10949         if (INTEL_GEN(dev_priv) >= 9)
10950                 skl_write_cursor_wm(plane, crtc_state);
10951
10952         if (plane->cursor.base != base ||
10953             plane->cursor.size != fbc_ctl ||
10954             plane->cursor.cntl != cntl) {
10955                 if (HAS_CUR_FBC(dev_priv))
10956                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10957                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10958                 I915_WRITE_FW(CURPOS(pipe), pos);
10959                 I915_WRITE_FW(CURBASE(pipe), base);
10960
10961                 plane->cursor.base = base;
10962                 plane->cursor.size = fbc_ctl;
10963                 plane->cursor.cntl = cntl;
10964         } else {
10965                 I915_WRITE_FW(CURPOS(pipe), pos);
10966                 I915_WRITE_FW(CURBASE(pipe), base);
10967         }
10968
10969         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10970 }
10971
10972 static void i9xx_disable_cursor(struct intel_plane *plane,
10973                                 const struct intel_crtc_state *crtc_state)
10974 {
10975         i9xx_update_cursor(plane, crtc_state, NULL);
10976 }
10977
10978 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10979                                      enum pipe *pipe)
10980 {
10981         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10982         enum intel_display_power_domain power_domain;
10983         intel_wakeref_t wakeref;
10984         bool ret;
10985         u32 val;
10986
10987         /*
10988          * Not 100% correct for planes that can move between pipes,
10989          * but that's only the case for gen2-3 which don't have any
10990          * display power wells.
10991          */
10992         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10993         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10994         if (!wakeref)
10995                 return false;
10996
10997         val = I915_READ(CURCNTR(plane->pipe));
10998
10999         ret = val & MCURSOR_MODE;
11000
11001         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11002                 *pipe = plane->pipe;
11003         else
11004                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11005                         MCURSOR_PIPE_SELECT_SHIFT;
11006
11007         intel_display_power_put(dev_priv, power_domain, wakeref);
11008
11009         return ret;
11010 }
11011
11012 /* VESA 640x480x72Hz mode to set on the pipe */
11013 static const struct drm_display_mode load_detect_mode = {
11014         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11015                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11016 };
11017
11018 struct drm_framebuffer *
11019 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11020                          struct drm_mode_fb_cmd2 *mode_cmd)
11021 {
11022         struct intel_framebuffer *intel_fb;
11023         int ret;
11024
11025         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11026         if (!intel_fb)
11027                 return ERR_PTR(-ENOMEM);
11028
11029         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11030         if (ret)
11031                 goto err;
11032
11033         return &intel_fb->base;
11034
11035 err:
11036         kfree(intel_fb);
11037         return ERR_PTR(ret);
11038 }
11039
11040 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11041                                         struct drm_crtc *crtc)
11042 {
11043         struct drm_plane *plane;
11044         struct drm_plane_state *plane_state;
11045         int ret, i;
11046
11047         ret = drm_atomic_add_affected_planes(state, crtc);
11048         if (ret)
11049                 return ret;
11050
11051         for_each_new_plane_in_state(state, plane, plane_state, i) {
11052                 if (plane_state->crtc != crtc)
11053                         continue;
11054
11055                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11056                 if (ret)
11057                         return ret;
11058
11059                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11060         }
11061
11062         return 0;
11063 }
11064
11065 int intel_get_load_detect_pipe(struct drm_connector *connector,
11066                                const struct drm_display_mode *mode,
11067                                struct intel_load_detect_pipe *old,
11068                                struct drm_modeset_acquire_ctx *ctx)
11069 {
11070         struct intel_crtc *intel_crtc;
11071         struct intel_encoder *intel_encoder =
11072                 intel_attached_encoder(connector);
11073         struct drm_crtc *possible_crtc;
11074         struct drm_encoder *encoder = &intel_encoder->base;
11075         struct drm_crtc *crtc = NULL;
11076         struct drm_device *dev = encoder->dev;
11077         struct drm_i915_private *dev_priv = to_i915(dev);
11078         struct drm_mode_config *config = &dev->mode_config;
11079         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11080         struct drm_connector_state *connector_state;
11081         struct intel_crtc_state *crtc_state;
11082         int ret, i = -1;
11083
11084         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11085                       connector->base.id, connector->name,
11086                       encoder->base.id, encoder->name);
11087
11088         old->restore_state = NULL;
11089
11090         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11091
11092         /*
11093          * Algorithm gets a little messy:
11094          *
11095          *   - if the connector already has an assigned crtc, use it (but make
11096          *     sure it's on first)
11097          *
11098          *   - try to find the first unused crtc that can drive this connector,
11099          *     and use that if we find one
11100          */
11101
11102         /* See if we already have a CRTC for this connector */
11103         if (connector->state->crtc) {
11104                 crtc = connector->state->crtc;
11105
11106                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11107                 if (ret)
11108                         goto fail;
11109
11110                 /* Make sure the crtc and connector are running */
11111                 goto found;
11112         }
11113
11114         /* Find an unused one (if possible) */
11115         for_each_crtc(dev, possible_crtc) {
11116                 i++;
11117                 if (!(encoder->possible_crtcs & (1 << i)))
11118                         continue;
11119
11120                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11121                 if (ret)
11122                         goto fail;
11123
11124                 if (possible_crtc->state->enable) {
11125                         drm_modeset_unlock(&possible_crtc->mutex);
11126                         continue;
11127                 }
11128
11129                 crtc = possible_crtc;
11130                 break;
11131         }
11132
11133         /*
11134          * If we didn't find an unused CRTC, don't use any.
11135          */
11136         if (!crtc) {
11137                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11138                 ret = -ENODEV;
11139                 goto fail;
11140         }
11141
11142 found:
11143         intel_crtc = to_intel_crtc(crtc);
11144
11145         state = drm_atomic_state_alloc(dev);
11146         restore_state = drm_atomic_state_alloc(dev);
11147         if (!state || !restore_state) {
11148                 ret = -ENOMEM;
11149                 goto fail;
11150         }
11151
11152         state->acquire_ctx = ctx;
11153         restore_state->acquire_ctx = ctx;
11154
11155         connector_state = drm_atomic_get_connector_state(state, connector);
11156         if (IS_ERR(connector_state)) {
11157                 ret = PTR_ERR(connector_state);
11158                 goto fail;
11159         }
11160
11161         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11162         if (ret)
11163                 goto fail;
11164
11165         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11166         if (IS_ERR(crtc_state)) {
11167                 ret = PTR_ERR(crtc_state);
11168                 goto fail;
11169         }
11170
11171         crtc_state->base.active = crtc_state->base.enable = true;
11172
11173         if (!mode)
11174                 mode = &load_detect_mode;
11175
11176         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
11177         if (ret)
11178                 goto fail;
11179
11180         ret = intel_modeset_disable_planes(state, crtc);
11181         if (ret)
11182                 goto fail;
11183
11184         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11185         if (!ret)
11186                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11187         if (!ret)
11188                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11189         if (ret) {
11190                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11191                 goto fail;
11192         }
11193
11194         ret = drm_atomic_commit(state);
11195         if (ret) {
11196                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11197                 goto fail;
11198         }
11199
11200         old->restore_state = restore_state;
11201         drm_atomic_state_put(state);
11202
11203         /* let the connector get through one full cycle before testing */
11204         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11205         return true;
11206
11207 fail:
11208         if (state) {
11209                 drm_atomic_state_put(state);
11210                 state = NULL;
11211         }
11212         if (restore_state) {
11213                 drm_atomic_state_put(restore_state);
11214                 restore_state = NULL;
11215         }
11216
11217         if (ret == -EDEADLK)
11218                 return ret;
11219
11220         return false;
11221 }
11222
11223 void intel_release_load_detect_pipe(struct drm_connector *connector,
11224                                     struct intel_load_detect_pipe *old,
11225                                     struct drm_modeset_acquire_ctx *ctx)
11226 {
11227         struct intel_encoder *intel_encoder =
11228                 intel_attached_encoder(connector);
11229         struct drm_encoder *encoder = &intel_encoder->base;
11230         struct drm_atomic_state *state = old->restore_state;
11231         int ret;
11232
11233         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11234                       connector->base.id, connector->name,
11235                       encoder->base.id, encoder->name);
11236
11237         if (!state)
11238                 return;
11239
11240         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11241         if (ret)
11242                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11243         drm_atomic_state_put(state);
11244 }
11245
11246 static int i9xx_pll_refclk(struct drm_device *dev,
11247                            const struct intel_crtc_state *pipe_config)
11248 {
11249         struct drm_i915_private *dev_priv = to_i915(dev);
11250         u32 dpll = pipe_config->dpll_hw_state.dpll;
11251
11252         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11253                 return dev_priv->vbt.lvds_ssc_freq;
11254         else if (HAS_PCH_SPLIT(dev_priv))
11255                 return 120000;
11256         else if (!IS_GEN(dev_priv, 2))
11257                 return 96000;
11258         else
11259                 return 48000;
11260 }
11261
11262 /* Returns the clock of the currently programmed mode of the given pipe. */
11263 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11264                                 struct intel_crtc_state *pipe_config)
11265 {
11266         struct drm_device *dev = crtc->base.dev;
11267         struct drm_i915_private *dev_priv = to_i915(dev);
11268         enum pipe pipe = crtc->pipe;
11269         u32 dpll = pipe_config->dpll_hw_state.dpll;
11270         u32 fp;
11271         struct dpll clock;
11272         int port_clock;
11273         int refclk = i9xx_pll_refclk(dev, pipe_config);
11274
11275         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11276                 fp = pipe_config->dpll_hw_state.fp0;
11277         else
11278                 fp = pipe_config->dpll_hw_state.fp1;
11279
11280         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11281         if (IS_PINEVIEW(dev_priv)) {
11282                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11283                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11284         } else {
11285                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11286                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11287         }
11288
11289         if (!IS_GEN(dev_priv, 2)) {
11290                 if (IS_PINEVIEW(dev_priv))
11291                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11292                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11293                 else
11294                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11295                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11296
11297                 switch (dpll & DPLL_MODE_MASK) {
11298                 case DPLLB_MODE_DAC_SERIAL:
11299                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11300                                 5 : 10;
11301                         break;
11302                 case DPLLB_MODE_LVDS:
11303                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11304                                 7 : 14;
11305                         break;
11306                 default:
11307                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11308                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11309                         return;
11310                 }
11311
11312                 if (IS_PINEVIEW(dev_priv))
11313                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11314                 else
11315                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11316         } else {
11317                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11318                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11319
11320                 if (is_lvds) {
11321                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11322                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11323
11324                         if (lvds & LVDS_CLKB_POWER_UP)
11325                                 clock.p2 = 7;
11326                         else
11327                                 clock.p2 = 14;
11328                 } else {
11329                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11330                                 clock.p1 = 2;
11331                         else {
11332                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11333                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11334                         }
11335                         if (dpll & PLL_P2_DIVIDE_BY_4)
11336                                 clock.p2 = 4;
11337                         else
11338                                 clock.p2 = 2;
11339                 }
11340
11341                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11342         }
11343
11344         /*
11345          * This value includes pixel_multiplier. We will use
11346          * port_clock to compute adjusted_mode.crtc_clock in the
11347          * encoder's get_config() function.
11348          */
11349         pipe_config->port_clock = port_clock;
11350 }
11351
11352 int intel_dotclock_calculate(int link_freq,
11353                              const struct intel_link_m_n *m_n)
11354 {
11355         /*
11356          * The calculation for the data clock is:
11357          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11358          * But we want to avoid losing precison if possible, so:
11359          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11360          *
11361          * and the link clock is simpler:
11362          * link_clock = (m * link_clock) / n
11363          */
11364
11365         if (!m_n->link_n)
11366                 return 0;
11367
11368         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11369 }
11370
11371 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11372                                    struct intel_crtc_state *pipe_config)
11373 {
11374         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11375
11376         /* read out port_clock from the DPLL */
11377         i9xx_crtc_clock_get(crtc, pipe_config);
11378
11379         /*
11380          * In case there is an active pipe without active ports,
11381          * we may need some idea for the dotclock anyway.
11382          * Calculate one based on the FDI configuration.
11383          */
11384         pipe_config->base.adjusted_mode.crtc_clock =
11385                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11386                                          &pipe_config->fdi_m_n);
11387 }
11388
11389 /* Returns the currently programmed mode of the given encoder. */
11390 struct drm_display_mode *
11391 intel_encoder_current_mode(struct intel_encoder *encoder)
11392 {
11393         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11394         struct intel_crtc_state *crtc_state;
11395         struct drm_display_mode *mode;
11396         struct intel_crtc *crtc;
11397         enum pipe pipe;
11398
11399         if (!encoder->get_hw_state(encoder, &pipe))
11400                 return NULL;
11401
11402         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11403
11404         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11405         if (!mode)
11406                 return NULL;
11407
11408         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
11409         if (!crtc_state) {
11410                 kfree(mode);
11411                 return NULL;
11412         }
11413
11414         crtc_state->base.crtc = &crtc->base;
11415
11416         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11417                 kfree(crtc_state);
11418                 kfree(mode);
11419                 return NULL;
11420         }
11421
11422         encoder->get_config(encoder, crtc_state);
11423
11424         intel_mode_from_pipe_config(mode, crtc_state);
11425
11426         kfree(crtc_state);
11427
11428         return mode;
11429 }
11430
11431 static void intel_crtc_destroy(struct drm_crtc *crtc)
11432 {
11433         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11434
11435         drm_crtc_cleanup(crtc);
11436         kfree(intel_crtc);
11437 }
11438
11439 /**
11440  * intel_wm_need_update - Check whether watermarks need updating
11441  * @cur: current plane state
11442  * @new: new plane state
11443  *
11444  * Check current plane state versus the new one to determine whether
11445  * watermarks need to be recalculated.
11446  *
11447  * Returns true or false.
11448  */
11449 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11450                                  struct intel_plane_state *new)
11451 {
11452         /* Update watermarks on tiling or size changes. */
11453         if (new->base.visible != cur->base.visible)
11454                 return true;
11455
11456         if (!cur->base.fb || !new->base.fb)
11457                 return false;
11458
11459         if (cur->base.fb->modifier != new->base.fb->modifier ||
11460             cur->base.rotation != new->base.rotation ||
11461             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
11462             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
11463             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
11464             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
11465                 return true;
11466
11467         return false;
11468 }
11469
11470 static bool needs_scaling(const struct intel_plane_state *state)
11471 {
11472         int src_w = drm_rect_width(&state->base.src) >> 16;
11473         int src_h = drm_rect_height(&state->base.src) >> 16;
11474         int dst_w = drm_rect_width(&state->base.dst);
11475         int dst_h = drm_rect_height(&state->base.dst);
11476
11477         return (src_w != dst_w || src_h != dst_h);
11478 }
11479
11480 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11481                                     struct intel_crtc_state *crtc_state,
11482                                     const struct intel_plane_state *old_plane_state,
11483                                     struct intel_plane_state *plane_state)
11484 {
11485         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11486         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
11487         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11488         bool mode_changed = needs_modeset(crtc_state);
11489         bool was_crtc_enabled = old_crtc_state->base.active;
11490         bool is_crtc_enabled = crtc_state->base.active;
11491         bool turn_off, turn_on, visible, was_visible;
11492         struct drm_framebuffer *fb = plane_state->base.fb;
11493         int ret;
11494
11495         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
11496                 ret = skl_update_scaler_plane(crtc_state, plane_state);
11497                 if (ret)
11498                         return ret;
11499         }
11500
11501         was_visible = old_plane_state->base.visible;
11502         visible = plane_state->base.visible;
11503
11504         if (!was_crtc_enabled && WARN_ON(was_visible))
11505                 was_visible = false;
11506
11507         /*
11508          * Visibility is calculated as if the crtc was on, but
11509          * after scaler setup everything depends on it being off
11510          * when the crtc isn't active.
11511          *
11512          * FIXME this is wrong for watermarks. Watermarks should also
11513          * be computed as if the pipe would be active. Perhaps move
11514          * per-plane wm computation to the .check_plane() hook, and
11515          * only combine the results from all planes in the current place?
11516          */
11517         if (!is_crtc_enabled) {
11518                 plane_state->base.visible = visible = false;
11519                 crtc_state->active_planes &= ~BIT(plane->id);
11520                 crtc_state->data_rate[plane->id] = 0;
11521         }
11522
11523         if (!was_visible && !visible)
11524                 return 0;
11525
11526         if (fb != old_plane_state->base.fb)
11527                 crtc_state->fb_changed = true;
11528
11529         turn_off = was_visible && (!visible || mode_changed);
11530         turn_on = visible && (!was_visible || mode_changed);
11531
11532         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
11533                          crtc->base.base.id, crtc->base.name,
11534                          plane->base.base.id, plane->base.name,
11535                          fb ? fb->base.id : -1);
11536
11537         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
11538                          plane->base.base.id, plane->base.name,
11539                          was_visible, visible,
11540                          turn_off, turn_on, mode_changed);
11541
11542         if (turn_on) {
11543                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11544                         crtc_state->update_wm_pre = true;
11545
11546                 /* must disable cxsr around plane enable/disable */
11547                 if (plane->id != PLANE_CURSOR)
11548                         crtc_state->disable_cxsr = true;
11549         } else if (turn_off) {
11550                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11551                         crtc_state->update_wm_post = true;
11552
11553                 /* must disable cxsr around plane enable/disable */
11554                 if (plane->id != PLANE_CURSOR)
11555                         crtc_state->disable_cxsr = true;
11556         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
11557                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
11558                         /* FIXME bollocks */
11559                         crtc_state->update_wm_pre = true;
11560                         crtc_state->update_wm_post = true;
11561                 }
11562         }
11563
11564         if (visible || was_visible)
11565                 crtc_state->fb_bits |= plane->frontbuffer_bit;
11566
11567         /*
11568          * ILK/SNB DVSACNTR/Sprite Enable
11569          * IVB SPR_CTL/Sprite Enable
11570          * "When in Self Refresh Big FIFO mode, a write to enable the
11571          *  plane will be internally buffered and delayed while Big FIFO
11572          *  mode is exiting."
11573          *
11574          * Which means that enabling the sprite can take an extra frame
11575          * when we start in big FIFO mode (LP1+). Thus we need to drop
11576          * down to LP0 and wait for vblank in order to make sure the
11577          * sprite gets enabled on the next vblank after the register write.
11578          * Doing otherwise would risk enabling the sprite one frame after
11579          * we've already signalled flip completion. We can resume LP1+
11580          * once the sprite has been enabled.
11581          *
11582          *
11583          * WaCxSRDisabledForSpriteScaling:ivb
11584          * IVB SPR_SCALE/Scaling Enable
11585          * "Low Power watermarks must be disabled for at least one
11586          *  frame before enabling sprite scaling, and kept disabled
11587          *  until sprite scaling is disabled."
11588          *
11589          * ILK/SNB DVSASCALE/Scaling Enable
11590          * "When in Self Refresh Big FIFO mode, scaling enable will be
11591          *  masked off while Big FIFO mode is exiting."
11592          *
11593          * Despite the w/a only being listed for IVB we assume that
11594          * the ILK/SNB note has similar ramifications, hence we apply
11595          * the w/a on all three platforms.
11596          *
11597          * With experimental results seems this is needed also for primary
11598          * plane, not only sprite plane.
11599          */
11600         if (plane->id != PLANE_CURSOR &&
11601             (IS_GEN_RANGE(dev_priv, 5, 6) ||
11602              IS_IVYBRIDGE(dev_priv)) &&
11603             (turn_on || (!needs_scaling(old_plane_state) &&
11604                          needs_scaling(plane_state))))
11605                 crtc_state->disable_lp_wm = true;
11606
11607         return 0;
11608 }
11609
11610 static bool encoders_cloneable(const struct intel_encoder *a,
11611                                const struct intel_encoder *b)
11612 {
11613         /* masks could be asymmetric, so check both ways */
11614         return a == b || (a->cloneable & (1 << b->type) &&
11615                           b->cloneable & (1 << a->type));
11616 }
11617
11618 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11619                                          struct intel_crtc *crtc,
11620                                          struct intel_encoder *encoder)
11621 {
11622         struct intel_encoder *source_encoder;
11623         struct drm_connector *connector;
11624         struct drm_connector_state *connector_state;
11625         int i;
11626
11627         for_each_new_connector_in_state(state, connector, connector_state, i) {
11628                 if (connector_state->crtc != &crtc->base)
11629                         continue;
11630
11631                 source_encoder =
11632                         to_intel_encoder(connector_state->best_encoder);
11633                 if (!encoders_cloneable(encoder, source_encoder))
11634                         return false;
11635         }
11636
11637         return true;
11638 }
11639
11640 static int icl_add_linked_planes(struct intel_atomic_state *state)
11641 {
11642         struct intel_plane *plane, *linked;
11643         struct intel_plane_state *plane_state, *linked_plane_state;
11644         int i;
11645
11646         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11647                 linked = plane_state->linked_plane;
11648
11649                 if (!linked)
11650                         continue;
11651
11652                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
11653                 if (IS_ERR(linked_plane_state))
11654                         return PTR_ERR(linked_plane_state);
11655
11656                 WARN_ON(linked_plane_state->linked_plane != plane);
11657                 WARN_ON(linked_plane_state->slave == plane_state->slave);
11658         }
11659
11660         return 0;
11661 }
11662
11663 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
11664 {
11665         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11666         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11667         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
11668         struct intel_plane *plane, *linked;
11669         struct intel_plane_state *plane_state;
11670         int i;
11671
11672         if (INTEL_GEN(dev_priv) < 11)
11673                 return 0;
11674
11675         /*
11676          * Destroy all old plane links and make the slave plane invisible
11677          * in the crtc_state->active_planes mask.
11678          */
11679         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11680                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
11681                         continue;
11682
11683                 plane_state->linked_plane = NULL;
11684                 if (plane_state->slave && !plane_state->base.visible) {
11685                         crtc_state->active_planes &= ~BIT(plane->id);
11686                         crtc_state->update_planes |= BIT(plane->id);
11687                 }
11688
11689                 plane_state->slave = false;
11690         }
11691
11692         if (!crtc_state->nv12_planes)
11693                 return 0;
11694
11695         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11696                 struct intel_plane_state *linked_state = NULL;
11697
11698                 if (plane->pipe != crtc->pipe ||
11699                     !(crtc_state->nv12_planes & BIT(plane->id)))
11700                         continue;
11701
11702                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
11703                         if (!icl_is_nv12_y_plane(linked->id))
11704                                 continue;
11705
11706                         if (crtc_state->active_planes & BIT(linked->id))
11707                                 continue;
11708
11709                         linked_state = intel_atomic_get_plane_state(state, linked);
11710                         if (IS_ERR(linked_state))
11711                                 return PTR_ERR(linked_state);
11712
11713                         break;
11714                 }
11715
11716                 if (!linked_state) {
11717                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
11718                                       hweight8(crtc_state->nv12_planes));
11719
11720                         return -EINVAL;
11721                 }
11722
11723                 plane_state->linked_plane = linked;
11724
11725                 linked_state->slave = true;
11726                 linked_state->linked_plane = plane;
11727                 crtc_state->active_planes |= BIT(linked->id);
11728                 crtc_state->update_planes |= BIT(linked->id);
11729                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
11730         }
11731
11732         return 0;
11733 }
11734
11735 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
11736 {
11737         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
11738         struct intel_atomic_state *state =
11739                 to_intel_atomic_state(new_crtc_state->base.state);
11740         const struct intel_crtc_state *old_crtc_state =
11741                 intel_atomic_get_old_crtc_state(state, crtc);
11742
11743         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
11744 }
11745
11746 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11747                                    struct drm_crtc_state *crtc_state)
11748 {
11749         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11750         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11751         struct intel_crtc_state *pipe_config =
11752                 to_intel_crtc_state(crtc_state);
11753         int ret;
11754         bool mode_changed = needs_modeset(pipe_config);
11755
11756         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
11757             mode_changed && !crtc_state->active)
11758                 pipe_config->update_wm_post = true;
11759
11760         if (mode_changed && crtc_state->enable &&
11761             dev_priv->display.crtc_compute_clock &&
11762             !WARN_ON(pipe_config->shared_dpll)) {
11763                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11764                                                            pipe_config);
11765                 if (ret)
11766                         return ret;
11767         }
11768
11769         /*
11770          * May need to update pipe gamma enable bits
11771          * when C8 planes are getting enabled/disabled.
11772          */
11773         if (c8_planes_changed(pipe_config))
11774                 crtc_state->color_mgmt_changed = true;
11775
11776         if (mode_changed || pipe_config->update_pipe ||
11777             crtc_state->color_mgmt_changed) {
11778                 ret = intel_color_check(pipe_config);
11779                 if (ret)
11780                         return ret;
11781         }
11782
11783         ret = 0;
11784         if (dev_priv->display.compute_pipe_wm) {
11785                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
11786                 if (ret) {
11787                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
11788                         return ret;
11789                 }
11790         }
11791
11792         if (dev_priv->display.compute_intermediate_wm) {
11793                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
11794                         return 0;
11795
11796                 /*
11797                  * Calculate 'intermediate' watermarks that satisfy both the
11798                  * old state and the new state.  We can program these
11799                  * immediately.
11800                  */
11801                 ret = dev_priv->display.compute_intermediate_wm(pipe_config);
11802                 if (ret) {
11803                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11804                         return ret;
11805                 }
11806         }
11807
11808         if (INTEL_GEN(dev_priv) >= 9) {
11809                 if (mode_changed || pipe_config->update_pipe)
11810                         ret = skl_update_scaler_crtc(pipe_config);
11811
11812                 if (!ret)
11813                         ret = icl_check_nv12_planes(pipe_config);
11814                 if (!ret)
11815                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11816                                                             pipe_config);
11817                 if (!ret)
11818                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11819                                                          pipe_config);
11820         }
11821
11822         if (HAS_IPS(dev_priv))
11823                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11824
11825         return ret;
11826 }
11827
11828 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11829         .atomic_check = intel_crtc_atomic_check,
11830 };
11831
11832 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11833 {
11834         struct intel_connector *connector;
11835         struct drm_connector_list_iter conn_iter;
11836
11837         drm_connector_list_iter_begin(dev, &conn_iter);
11838         for_each_intel_connector_iter(connector, &conn_iter) {
11839                 if (connector->base.state->crtc)
11840                         drm_connector_put(&connector->base);
11841
11842                 if (connector->base.encoder) {
11843                         connector->base.state->best_encoder =
11844                                 connector->base.encoder;
11845                         connector->base.state->crtc =
11846                                 connector->base.encoder->crtc;
11847
11848                         drm_connector_get(&connector->base);
11849                 } else {
11850                         connector->base.state->best_encoder = NULL;
11851                         connector->base.state->crtc = NULL;
11852                 }
11853         }
11854         drm_connector_list_iter_end(&conn_iter);
11855 }
11856
11857 static int
11858 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11859                       struct intel_crtc_state *pipe_config)
11860 {
11861         struct drm_connector *connector = conn_state->connector;
11862         const struct drm_display_info *info = &connector->display_info;
11863         int bpp;
11864
11865         switch (conn_state->max_bpc) {
11866         case 6 ... 7:
11867                 bpp = 6 * 3;
11868                 break;
11869         case 8 ... 9:
11870                 bpp = 8 * 3;
11871                 break;
11872         case 10 ... 11:
11873                 bpp = 10 * 3;
11874                 break;
11875         case 12:
11876                 bpp = 12 * 3;
11877                 break;
11878         default:
11879                 return -EINVAL;
11880         }
11881
11882         if (bpp < pipe_config->pipe_bpp) {
11883                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11884                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11885                               connector->base.id, connector->name,
11886                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11887                               pipe_config->pipe_bpp);
11888
11889                 pipe_config->pipe_bpp = bpp;
11890         }
11891
11892         return 0;
11893 }
11894
11895 static int
11896 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11897                           struct intel_crtc_state *pipe_config)
11898 {
11899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11900         struct drm_atomic_state *state = pipe_config->base.state;
11901         struct drm_connector *connector;
11902         struct drm_connector_state *connector_state;
11903         int bpp, i;
11904
11905         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11906             IS_CHERRYVIEW(dev_priv)))
11907                 bpp = 10*3;
11908         else if (INTEL_GEN(dev_priv) >= 5)
11909                 bpp = 12*3;
11910         else
11911                 bpp = 8*3;
11912
11913         pipe_config->pipe_bpp = bpp;
11914
11915         /* Clamp display bpp to connector max bpp */
11916         for_each_new_connector_in_state(state, connector, connector_state, i) {
11917                 int ret;
11918
11919                 if (connector_state->crtc != &crtc->base)
11920                         continue;
11921
11922                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11923                 if (ret)
11924                         return ret;
11925         }
11926
11927         return 0;
11928 }
11929
11930 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11931 {
11932         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11933                       "type: 0x%x flags: 0x%x\n",
11934                       mode->crtc_clock,
11935                       mode->crtc_hdisplay, mode->crtc_hsync_start,
11936                       mode->crtc_hsync_end, mode->crtc_htotal,
11937                       mode->crtc_vdisplay, mode->crtc_vsync_start,
11938                       mode->crtc_vsync_end, mode->crtc_vtotal,
11939                       mode->type, mode->flags);
11940 }
11941
11942 static inline void
11943 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
11944                       const char *id, unsigned int lane_count,
11945                       const struct intel_link_m_n *m_n)
11946 {
11947         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11948                       id, lane_count,
11949                       m_n->gmch_m, m_n->gmch_n,
11950                       m_n->link_m, m_n->link_n, m_n->tu);
11951 }
11952
11953 static void
11954 intel_dump_infoframe(struct drm_i915_private *dev_priv,
11955                      const union hdmi_infoframe *frame)
11956 {
11957         if ((drm_debug & DRM_UT_KMS) == 0)
11958                 return;
11959
11960         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
11961 }
11962
11963 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11964
11965 static const char * const output_type_str[] = {
11966         OUTPUT_TYPE(UNUSED),
11967         OUTPUT_TYPE(ANALOG),
11968         OUTPUT_TYPE(DVO),
11969         OUTPUT_TYPE(SDVO),
11970         OUTPUT_TYPE(LVDS),
11971         OUTPUT_TYPE(TVOUT),
11972         OUTPUT_TYPE(HDMI),
11973         OUTPUT_TYPE(DP),
11974         OUTPUT_TYPE(EDP),
11975         OUTPUT_TYPE(DSI),
11976         OUTPUT_TYPE(DDI),
11977         OUTPUT_TYPE(DP_MST),
11978 };
11979
11980 #undef OUTPUT_TYPE
11981
11982 static void snprintf_output_types(char *buf, size_t len,
11983                                   unsigned int output_types)
11984 {
11985         char *str = buf;
11986         int i;
11987
11988         str[0] = '\0';
11989
11990         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
11991                 int r;
11992
11993                 if ((output_types & BIT(i)) == 0)
11994                         continue;
11995
11996                 r = snprintf(str, len, "%s%s",
11997                              str != buf ? "," : "", output_type_str[i]);
11998                 if (r >= len)
11999                         break;
12000                 str += r;
12001                 len -= r;
12002
12003                 output_types &= ~BIT(i);
12004         }
12005
12006         WARN_ON_ONCE(output_types != 0);
12007 }
12008
12009 static const char * const output_format_str[] = {
12010         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12011         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12012         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12013         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12014 };
12015
12016 static const char *output_formats(enum intel_output_format format)
12017 {
12018         if (format >= ARRAY_SIZE(output_format_str))
12019                 format = INTEL_OUTPUT_FORMAT_INVALID;
12020         return output_format_str[format];
12021 }
12022
12023 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12024 {
12025         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
12026         const struct drm_framebuffer *fb = plane_state->base.fb;
12027         struct drm_format_name_buf format_name;
12028
12029         if (!fb) {
12030                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12031                               plane->base.base.id, plane->base.name,
12032                               yesno(plane_state->base.visible));
12033                 return;
12034         }
12035
12036         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12037                       plane->base.base.id, plane->base.name,
12038                       fb->base.id, fb->width, fb->height,
12039                       drm_get_format_name(fb->format->format, &format_name),
12040                       yesno(plane_state->base.visible));
12041         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12042                       plane_state->base.rotation, plane_state->scaler_id);
12043         if (plane_state->base.visible)
12044                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12045                               DRM_RECT_FP_ARG(&plane_state->base.src),
12046                               DRM_RECT_ARG(&plane_state->base.dst));
12047 }
12048
12049 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12050                                    struct intel_atomic_state *state,
12051                                    const char *context)
12052 {
12053         struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
12054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12055         const struct intel_plane_state *plane_state;
12056         struct intel_plane *plane;
12057         char buf[64];
12058         int i;
12059
12060         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12061                       crtc->base.base.id, crtc->base.name,
12062                       yesno(pipe_config->base.enable), context);
12063
12064         if (!pipe_config->base.enable)
12065                 goto dump_planes;
12066
12067         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12068         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12069                       yesno(pipe_config->base.active),
12070                       buf, pipe_config->output_types,
12071                       output_formats(pipe_config->output_format));
12072
12073         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12074                       transcoder_name(pipe_config->cpu_transcoder),
12075                       pipe_config->pipe_bpp, pipe_config->dither);
12076
12077         if (pipe_config->has_pch_encoder)
12078                 intel_dump_m_n_config(pipe_config, "fdi",
12079                                       pipe_config->fdi_lanes,
12080                                       &pipe_config->fdi_m_n);
12081
12082         if (intel_crtc_has_dp_encoder(pipe_config)) {
12083                 intel_dump_m_n_config(pipe_config, "dp m_n",
12084                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12085                 if (pipe_config->has_drrs)
12086                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12087                                               pipe_config->lane_count,
12088                                               &pipe_config->dp_m2_n2);
12089         }
12090
12091         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12092                       pipe_config->has_audio, pipe_config->has_infoframe,
12093                       pipe_config->infoframes.enable);
12094
12095         if (pipe_config->infoframes.enable &
12096             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12097                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12098         if (pipe_config->infoframes.enable &
12099             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12100                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12101         if (pipe_config->infoframes.enable &
12102             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12103                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12104         if (pipe_config->infoframes.enable &
12105             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12106                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12107
12108         DRM_DEBUG_KMS("requested mode:\n");
12109         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12110         DRM_DEBUG_KMS("adjusted mode:\n");
12111         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12112         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12113         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12114                       pipe_config->port_clock,
12115                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12116                       pipe_config->pixel_rate);
12117
12118         if (INTEL_GEN(dev_priv) >= 9)
12119                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12120                               crtc->num_scalers,
12121                               pipe_config->scaler_state.scaler_users,
12122                               pipe_config->scaler_state.scaler_id);
12123
12124         if (HAS_GMCH(dev_priv))
12125                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12126                               pipe_config->gmch_pfit.control,
12127                               pipe_config->gmch_pfit.pgm_ratios,
12128                               pipe_config->gmch_pfit.lvds_border_bits);
12129         else
12130                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12131                               pipe_config->pch_pfit.pos,
12132                               pipe_config->pch_pfit.size,
12133                               enableddisabled(pipe_config->pch_pfit.enabled),
12134                               yesno(pipe_config->pch_pfit.force_thru));
12135
12136         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12137                       pipe_config->ips_enabled, pipe_config->double_wide);
12138
12139         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12140
12141 dump_planes:
12142         if (!state)
12143                 return;
12144
12145         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12146                 if (plane->pipe == crtc->pipe)
12147                         intel_dump_plane_state(plane_state);
12148         }
12149 }
12150
12151 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12152 {
12153         struct drm_device *dev = state->base.dev;
12154         struct drm_connector *connector;
12155         struct drm_connector_list_iter conn_iter;
12156         unsigned int used_ports = 0;
12157         unsigned int used_mst_ports = 0;
12158         bool ret = true;
12159
12160         /*
12161          * Walk the connector list instead of the encoder
12162          * list to detect the problem on ddi platforms
12163          * where there's just one encoder per digital port.
12164          */
12165         drm_connector_list_iter_begin(dev, &conn_iter);
12166         drm_for_each_connector_iter(connector, &conn_iter) {
12167                 struct drm_connector_state *connector_state;
12168                 struct intel_encoder *encoder;
12169
12170                 connector_state =
12171                         drm_atomic_get_new_connector_state(&state->base,
12172                                                            connector);
12173                 if (!connector_state)
12174                         connector_state = connector->state;
12175
12176                 if (!connector_state->best_encoder)
12177                         continue;
12178
12179                 encoder = to_intel_encoder(connector_state->best_encoder);
12180
12181                 WARN_ON(!connector_state->crtc);
12182
12183                 switch (encoder->type) {
12184                         unsigned int port_mask;
12185                 case INTEL_OUTPUT_DDI:
12186                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12187                                 break;
12188                         /* else, fall through */
12189                 case INTEL_OUTPUT_DP:
12190                 case INTEL_OUTPUT_HDMI:
12191                 case INTEL_OUTPUT_EDP:
12192                         port_mask = 1 << encoder->port;
12193
12194                         /* the same port mustn't appear more than once */
12195                         if (used_ports & port_mask)
12196                                 ret = false;
12197
12198                         used_ports |= port_mask;
12199                         break;
12200                 case INTEL_OUTPUT_DP_MST:
12201                         used_mst_ports |=
12202                                 1 << encoder->port;
12203                         break;
12204                 default:
12205                         break;
12206                 }
12207         }
12208         drm_connector_list_iter_end(&conn_iter);
12209
12210         /* can't mix MST and SST/HDMI on the same port */
12211         if (used_ports & used_mst_ports)
12212                 return false;
12213
12214         return ret;
12215 }
12216
12217 static int
12218 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12219 {
12220         struct drm_i915_private *dev_priv =
12221                 to_i915(crtc_state->base.crtc->dev);
12222         struct intel_crtc_state *saved_state;
12223
12224         saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL);
12225         if (!saved_state)
12226                 return -ENOMEM;
12227
12228         /* FIXME: before the switch to atomic started, a new pipe_config was
12229          * kzalloc'd. Code that depends on any field being zero should be
12230          * fixed, so that the crtc_state can be safely duplicated. For now,
12231          * only fields that are know to not cause problems are preserved. */
12232
12233         saved_state->scaler_state = crtc_state->scaler_state;
12234         saved_state->shared_dpll = crtc_state->shared_dpll;
12235         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12236         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12237                sizeof(saved_state->icl_port_dplls));
12238         saved_state->crc_enabled = crtc_state->crc_enabled;
12239         if (IS_G4X(dev_priv) ||
12240             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12241                 saved_state->wm = crtc_state->wm;
12242
12243         /* Keep base drm_crtc_state intact, only clear our extended struct */
12244         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
12245         memcpy(&crtc_state->base + 1, &saved_state->base + 1,
12246                sizeof(*crtc_state) - sizeof(crtc_state->base));
12247
12248         kfree(saved_state);
12249         return 0;
12250 }
12251
12252 static int
12253 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12254 {
12255         struct drm_crtc *crtc = pipe_config->base.crtc;
12256         struct drm_atomic_state *state = pipe_config->base.state;
12257         struct intel_encoder *encoder;
12258         struct drm_connector *connector;
12259         struct drm_connector_state *connector_state;
12260         int base_bpp, ret;
12261         int i;
12262         bool retry = true;
12263
12264         ret = clear_intel_crtc_state(pipe_config);
12265         if (ret)
12266                 return ret;
12267
12268         pipe_config->cpu_transcoder =
12269                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12270
12271         /*
12272          * Sanitize sync polarity flags based on requested ones. If neither
12273          * positive or negative polarity is requested, treat this as meaning
12274          * negative polarity.
12275          */
12276         if (!(pipe_config->base.adjusted_mode.flags &
12277               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12278                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12279
12280         if (!(pipe_config->base.adjusted_mode.flags &
12281               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12282                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12283
12284         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12285                                         pipe_config);
12286         if (ret)
12287                 return ret;
12288
12289         base_bpp = pipe_config->pipe_bpp;
12290
12291         /*
12292          * Determine the real pipe dimensions. Note that stereo modes can
12293          * increase the actual pipe size due to the frame doubling and
12294          * insertion of additional space for blanks between the frame. This
12295          * is stored in the crtc timings. We use the requested mode to do this
12296          * computation to clearly distinguish it from the adjusted mode, which
12297          * can be changed by the connectors in the below retry loop.
12298          */
12299         drm_mode_get_hv_timing(&pipe_config->base.mode,
12300                                &pipe_config->pipe_src_w,
12301                                &pipe_config->pipe_src_h);
12302
12303         for_each_new_connector_in_state(state, connector, connector_state, i) {
12304                 if (connector_state->crtc != crtc)
12305                         continue;
12306
12307                 encoder = to_intel_encoder(connector_state->best_encoder);
12308
12309                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12310                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12311                         return -EINVAL;
12312                 }
12313
12314                 /*
12315                  * Determine output_types before calling the .compute_config()
12316                  * hooks so that the hooks can use this information safely.
12317                  */
12318                 if (encoder->compute_output_type)
12319                         pipe_config->output_types |=
12320                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12321                                                                  connector_state));
12322                 else
12323                         pipe_config->output_types |= BIT(encoder->type);
12324         }
12325
12326 encoder_retry:
12327         /* Ensure the port clock defaults are reset when retrying. */
12328         pipe_config->port_clock = 0;
12329         pipe_config->pixel_multiplier = 1;
12330
12331         /* Fill in default crtc timings, allow encoders to overwrite them. */
12332         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12333                               CRTC_STEREO_DOUBLE);
12334
12335         /* Pass our mode to the connectors and the CRTC to give them a chance to
12336          * adjust it according to limitations or connector properties, and also
12337          * a chance to reject the mode entirely.
12338          */
12339         for_each_new_connector_in_state(state, connector, connector_state, i) {
12340                 if (connector_state->crtc != crtc)
12341                         continue;
12342
12343                 encoder = to_intel_encoder(connector_state->best_encoder);
12344                 ret = encoder->compute_config(encoder, pipe_config,
12345                                               connector_state);
12346                 if (ret < 0) {
12347                         if (ret != -EDEADLK)
12348                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
12349                                               ret);
12350                         return ret;
12351                 }
12352         }
12353
12354         /* Set default port clock if not overwritten by the encoder. Needs to be
12355          * done afterwards in case the encoder adjusts the mode. */
12356         if (!pipe_config->port_clock)
12357                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12358                         * pipe_config->pixel_multiplier;
12359
12360         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12361         if (ret == -EDEADLK)
12362                 return ret;
12363         if (ret < 0) {
12364                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12365                 return ret;
12366         }
12367
12368         if (ret == RETRY) {
12369                 if (WARN(!retry, "loop in pipe configuration computation\n"))
12370                         return -EINVAL;
12371
12372                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12373                 retry = false;
12374                 goto encoder_retry;
12375         }
12376
12377         /* Dithering seems to not pass-through bits correctly when it should, so
12378          * only enable it on 6bpc panels and when its not a compliance
12379          * test requesting 6bpc video pattern.
12380          */
12381         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
12382                 !pipe_config->dither_force_disable;
12383         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12384                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12385
12386         return 0;
12387 }
12388
12389 bool intel_fuzzy_clock_check(int clock1, int clock2)
12390 {
12391         int diff;
12392
12393         if (clock1 == clock2)
12394                 return true;
12395
12396         if (!clock1 || !clock2)
12397                 return false;
12398
12399         diff = abs(clock1 - clock2);
12400
12401         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12402                 return true;
12403
12404         return false;
12405 }
12406
12407 static bool
12408 intel_compare_m_n(unsigned int m, unsigned int n,
12409                   unsigned int m2, unsigned int n2,
12410                   bool exact)
12411 {
12412         if (m == m2 && n == n2)
12413                 return true;
12414
12415         if (exact || !m || !n || !m2 || !n2)
12416                 return false;
12417
12418         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12419
12420         if (n > n2) {
12421                 while (n > n2) {
12422                         m2 <<= 1;
12423                         n2 <<= 1;
12424                 }
12425         } else if (n < n2) {
12426                 while (n < n2) {
12427                         m <<= 1;
12428                         n <<= 1;
12429                 }
12430         }
12431
12432         if (n != n2)
12433                 return false;
12434
12435         return intel_fuzzy_clock_check(m, m2);
12436 }
12437
12438 static bool
12439 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12440                        const struct intel_link_m_n *m2_n2,
12441                        bool exact)
12442 {
12443         return m_n->tu == m2_n2->tu &&
12444                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12445                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
12446                 intel_compare_m_n(m_n->link_m, m_n->link_n,
12447                                   m2_n2->link_m, m2_n2->link_n, exact);
12448 }
12449
12450 static bool
12451 intel_compare_infoframe(const union hdmi_infoframe *a,
12452                         const union hdmi_infoframe *b)
12453 {
12454         return memcmp(a, b, sizeof(*a)) == 0;
12455 }
12456
12457 static void
12458 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
12459                                bool fastset, const char *name,
12460                                const union hdmi_infoframe *a,
12461                                const union hdmi_infoframe *b)
12462 {
12463         if (fastset) {
12464                 if ((drm_debug & DRM_UT_KMS) == 0)
12465                         return;
12466
12467                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s infoframe", name);
12468                 drm_dbg(DRM_UT_KMS, "expected:");
12469                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
12470                 drm_dbg(DRM_UT_KMS, "found");
12471                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
12472         } else {
12473                 drm_err("mismatch in %s infoframe", name);
12474                 drm_err("expected:");
12475                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
12476                 drm_err("found");
12477                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
12478         }
12479 }
12480
12481 static void __printf(3, 4)
12482 pipe_config_mismatch(bool fastset, const char *name, const char *format, ...)
12483 {
12484         struct va_format vaf;
12485         va_list args;
12486
12487         va_start(args, format);
12488         vaf.fmt = format;
12489         vaf.va = &args;
12490
12491         if (fastset)
12492                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s %pV", name, &vaf);
12493         else
12494                 drm_err("mismatch in %s %pV", name, &vaf);
12495
12496         va_end(args);
12497 }
12498
12499 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
12500 {
12501         if (i915_modparams.fastboot != -1)
12502                 return i915_modparams.fastboot;
12503
12504         /* Enable fastboot by default on Skylake and newer */
12505         if (INTEL_GEN(dev_priv) >= 9)
12506                 return true;
12507
12508         /* Enable fastboot by default on VLV and CHV */
12509         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12510                 return true;
12511
12512         /* Disabled by default on all others */
12513         return false;
12514 }
12515
12516 static bool
12517 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
12518                           const struct intel_crtc_state *pipe_config,
12519                           bool fastset)
12520 {
12521         struct drm_i915_private *dev_priv = to_i915(current_config->base.crtc->dev);
12522         bool ret = true;
12523         bool fixup_inherited = fastset &&
12524                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
12525                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
12526
12527         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
12528                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
12529                 ret = false;
12530         }
12531
12532 #define PIPE_CONF_CHECK_X(name) do { \
12533         if (current_config->name != pipe_config->name) { \
12534                 pipe_config_mismatch(fastset, __stringify(name), \
12535                                      "(expected 0x%08x, found 0x%08x)\n", \
12536                                      current_config->name, \
12537                                      pipe_config->name); \
12538                 ret = false; \
12539         } \
12540 } while (0)
12541
12542 #define PIPE_CONF_CHECK_I(name) do { \
12543         if (current_config->name != pipe_config->name) { \
12544                 pipe_config_mismatch(fastset, __stringify(name), \
12545                                      "(expected %i, found %i)\n", \
12546                                      current_config->name, \
12547                                      pipe_config->name); \
12548                 ret = false; \
12549         } \
12550 } while (0)
12551
12552 #define PIPE_CONF_CHECK_BOOL(name) do { \
12553         if (current_config->name != pipe_config->name) { \
12554                 pipe_config_mismatch(fastset, __stringify(name), \
12555                                      "(expected %s, found %s)\n", \
12556                                      yesno(current_config->name), \
12557                                      yesno(pipe_config->name)); \
12558                 ret = false; \
12559         } \
12560 } while (0)
12561
12562 /*
12563  * Checks state where we only read out the enabling, but not the entire
12564  * state itself (like full infoframes or ELD for audio). These states
12565  * require a full modeset on bootup to fix up.
12566  */
12567 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
12568         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
12569                 PIPE_CONF_CHECK_BOOL(name); \
12570         } else { \
12571                 pipe_config_mismatch(fastset, __stringify(name), \
12572                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
12573                                      yesno(current_config->name), \
12574                                      yesno(pipe_config->name)); \
12575                 ret = false; \
12576         } \
12577 } while (0)
12578
12579 #define PIPE_CONF_CHECK_P(name) do { \
12580         if (current_config->name != pipe_config->name) { \
12581                 pipe_config_mismatch(fastset, __stringify(name), \
12582                                      "(expected %p, found %p)\n", \
12583                                      current_config->name, \
12584                                      pipe_config->name); \
12585                 ret = false; \
12586         } \
12587 } while (0)
12588
12589 #define PIPE_CONF_CHECK_M_N(name) do { \
12590         if (!intel_compare_link_m_n(&current_config->name, \
12591                                     &pipe_config->name,\
12592                                     !fastset)) { \
12593                 pipe_config_mismatch(fastset, __stringify(name), \
12594                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12595                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12596                                      current_config->name.tu, \
12597                                      current_config->name.gmch_m, \
12598                                      current_config->name.gmch_n, \
12599                                      current_config->name.link_m, \
12600                                      current_config->name.link_n, \
12601                                      pipe_config->name.tu, \
12602                                      pipe_config->name.gmch_m, \
12603                                      pipe_config->name.gmch_n, \
12604                                      pipe_config->name.link_m, \
12605                                      pipe_config->name.link_n); \
12606                 ret = false; \
12607         } \
12608 } while (0)
12609
12610 /* This is required for BDW+ where there is only one set of registers for
12611  * switching between high and low RR.
12612  * This macro can be used whenever a comparison has to be made between one
12613  * hw state and multiple sw state variables.
12614  */
12615 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
12616         if (!intel_compare_link_m_n(&current_config->name, \
12617                                     &pipe_config->name, !fastset) && \
12618             !intel_compare_link_m_n(&current_config->alt_name, \
12619                                     &pipe_config->name, !fastset)) { \
12620                 pipe_config_mismatch(fastset, __stringify(name), \
12621                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12622                                      "or tu %i gmch %i/%i link %i/%i, " \
12623                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12624                                      current_config->name.tu, \
12625                                      current_config->name.gmch_m, \
12626                                      current_config->name.gmch_n, \
12627                                      current_config->name.link_m, \
12628                                      current_config->name.link_n, \
12629                                      current_config->alt_name.tu, \
12630                                      current_config->alt_name.gmch_m, \
12631                                      current_config->alt_name.gmch_n, \
12632                                      current_config->alt_name.link_m, \
12633                                      current_config->alt_name.link_n, \
12634                                      pipe_config->name.tu, \
12635                                      pipe_config->name.gmch_m, \
12636                                      pipe_config->name.gmch_n, \
12637                                      pipe_config->name.link_m, \
12638                                      pipe_config->name.link_n); \
12639                 ret = false; \
12640         } \
12641 } while (0)
12642
12643 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
12644         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12645                 pipe_config_mismatch(fastset, __stringify(name), \
12646                                      "(%x) (expected %i, found %i)\n", \
12647                                      (mask), \
12648                                      current_config->name & (mask), \
12649                                      pipe_config->name & (mask)); \
12650                 ret = false; \
12651         } \
12652 } while (0)
12653
12654 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
12655         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12656                 pipe_config_mismatch(fastset, __stringify(name), \
12657                                      "(expected %i, found %i)\n", \
12658                                      current_config->name, \
12659                                      pipe_config->name); \
12660                 ret = false; \
12661         } \
12662 } while (0)
12663
12664 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
12665         if (!intel_compare_infoframe(&current_config->infoframes.name, \
12666                                      &pipe_config->infoframes.name)) { \
12667                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
12668                                                &current_config->infoframes.name, \
12669                                                &pipe_config->infoframes.name); \
12670                 ret = false; \
12671         } \
12672 } while (0)
12673
12674 #define PIPE_CONF_QUIRK(quirk) \
12675         ((current_config->quirks | pipe_config->quirks) & (quirk))
12676
12677         PIPE_CONF_CHECK_I(cpu_transcoder);
12678
12679         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
12680         PIPE_CONF_CHECK_I(fdi_lanes);
12681         PIPE_CONF_CHECK_M_N(fdi_m_n);
12682
12683         PIPE_CONF_CHECK_I(lane_count);
12684         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
12685
12686         if (INTEL_GEN(dev_priv) < 8) {
12687                 PIPE_CONF_CHECK_M_N(dp_m_n);
12688
12689                 if (current_config->has_drrs)
12690                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12691         } else
12692                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12693
12694         PIPE_CONF_CHECK_X(output_types);
12695
12696         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12697         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12698         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12699         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12700         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12701         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12702
12703         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12704         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12705         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12706         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12707         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12708         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12709
12710         PIPE_CONF_CHECK_I(pixel_multiplier);
12711         PIPE_CONF_CHECK_I(output_format);
12712         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
12713         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
12714             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12715                 PIPE_CONF_CHECK_BOOL(limited_color_range);
12716
12717         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
12718         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
12719         PIPE_CONF_CHECK_BOOL(has_infoframe);
12720
12721         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
12722
12723         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12724                               DRM_MODE_FLAG_INTERLACE);
12725
12726         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12727                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12728                                       DRM_MODE_FLAG_PHSYNC);
12729                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12730                                       DRM_MODE_FLAG_NHSYNC);
12731                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12732                                       DRM_MODE_FLAG_PVSYNC);
12733                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12734                                       DRM_MODE_FLAG_NVSYNC);
12735         }
12736
12737         PIPE_CONF_CHECK_X(gmch_pfit.control);
12738         /* pfit ratios are autocomputed by the hw on gen4+ */
12739         if (INTEL_GEN(dev_priv) < 4)
12740                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
12741         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12742
12743         /*
12744          * Changing the EDP transcoder input mux
12745          * (A_ONOFF vs. A_ON) requires a full modeset.
12746          */
12747         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
12748
12749         if (!fastset) {
12750                 PIPE_CONF_CHECK_I(pipe_src_w);
12751                 PIPE_CONF_CHECK_I(pipe_src_h);
12752
12753                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
12754                 if (current_config->pch_pfit.enabled) {
12755                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12756                         PIPE_CONF_CHECK_X(pch_pfit.size);
12757                 }
12758
12759                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12760                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
12761
12762                 PIPE_CONF_CHECK_X(gamma_mode);
12763                 if (IS_CHERRYVIEW(dev_priv))
12764                         PIPE_CONF_CHECK_X(cgm_mode);
12765                 else
12766                         PIPE_CONF_CHECK_X(csc_mode);
12767                 PIPE_CONF_CHECK_BOOL(gamma_enable);
12768                 PIPE_CONF_CHECK_BOOL(csc_enable);
12769         }
12770
12771         PIPE_CONF_CHECK_BOOL(double_wide);
12772
12773         PIPE_CONF_CHECK_P(shared_dpll);
12774         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12775         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12776         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12777         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12778         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12779         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12780         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12781         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12782         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12783         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
12784         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
12785         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
12786         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
12787         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
12788         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
12789         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
12790         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
12791         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
12792         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
12793         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
12794         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
12795         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
12796         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
12797         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
12798         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
12799         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
12800         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
12801         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
12802         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
12803         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
12804         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
12805
12806         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
12807         PIPE_CONF_CHECK_X(dsi_pll.div);
12808
12809         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
12810                 PIPE_CONF_CHECK_I(pipe_bpp);
12811
12812         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12813         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12814
12815         PIPE_CONF_CHECK_I(min_voltage_level);
12816
12817         PIPE_CONF_CHECK_X(infoframes.enable);
12818         PIPE_CONF_CHECK_X(infoframes.gcp);
12819         PIPE_CONF_CHECK_INFOFRAME(avi);
12820         PIPE_CONF_CHECK_INFOFRAME(spd);
12821         PIPE_CONF_CHECK_INFOFRAME(hdmi);
12822         PIPE_CONF_CHECK_INFOFRAME(drm);
12823
12824 #undef PIPE_CONF_CHECK_X
12825 #undef PIPE_CONF_CHECK_I
12826 #undef PIPE_CONF_CHECK_BOOL
12827 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
12828 #undef PIPE_CONF_CHECK_P
12829 #undef PIPE_CONF_CHECK_FLAGS
12830 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12831 #undef PIPE_CONF_QUIRK
12832
12833         return ret;
12834 }
12835
12836 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
12837                                            const struct intel_crtc_state *pipe_config)
12838 {
12839         if (pipe_config->has_pch_encoder) {
12840                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12841                                                             &pipe_config->fdi_m_n);
12842                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
12843
12844                 /*
12845                  * FDI already provided one idea for the dotclock.
12846                  * Yell if the encoder disagrees.
12847                  */
12848                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
12849                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12850                      fdi_dotclock, dotclock);
12851         }
12852 }
12853
12854 static void verify_wm_state(struct intel_crtc *crtc,
12855                             struct intel_crtc_state *new_crtc_state)
12856 {
12857         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12858         struct skl_hw_state {
12859                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
12860                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
12861                 struct skl_ddb_allocation ddb;
12862                 struct skl_pipe_wm wm;
12863         } *hw;
12864         struct skl_ddb_allocation *sw_ddb;
12865         struct skl_pipe_wm *sw_wm;
12866         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
12867         const enum pipe pipe = crtc->pipe;
12868         int plane, level, max_level = ilk_wm_max_level(dev_priv);
12869
12870         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->base.active)
12871                 return;
12872
12873         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
12874         if (!hw)
12875                 return;
12876
12877         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
12878         sw_wm = &new_crtc_state->wm.skl.optimal;
12879
12880         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
12881
12882         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
12883         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12884
12885         if (INTEL_GEN(dev_priv) >= 11 &&
12886             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
12887                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
12888                           sw_ddb->enabled_slices,
12889                           hw->ddb.enabled_slices);
12890
12891         /* planes */
12892         for_each_universal_plane(dev_priv, pipe, plane) {
12893                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12894
12895                 hw_plane_wm = &hw->wm.planes[plane];
12896                 sw_plane_wm = &sw_wm->planes[plane];
12897
12898                 /* Watermarks */
12899                 for (level = 0; level <= max_level; level++) {
12900                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12901                                                 &sw_plane_wm->wm[level]))
12902                                 continue;
12903
12904                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12905                                   pipe_name(pipe), plane + 1, level,
12906                                   sw_plane_wm->wm[level].plane_en,
12907                                   sw_plane_wm->wm[level].plane_res_b,
12908                                   sw_plane_wm->wm[level].plane_res_l,
12909                                   hw_plane_wm->wm[level].plane_en,
12910                                   hw_plane_wm->wm[level].plane_res_b,
12911                                   hw_plane_wm->wm[level].plane_res_l);
12912                 }
12913
12914                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12915                                          &sw_plane_wm->trans_wm)) {
12916                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12917                                   pipe_name(pipe), plane + 1,
12918                                   sw_plane_wm->trans_wm.plane_en,
12919                                   sw_plane_wm->trans_wm.plane_res_b,
12920                                   sw_plane_wm->trans_wm.plane_res_l,
12921                                   hw_plane_wm->trans_wm.plane_en,
12922                                   hw_plane_wm->trans_wm.plane_res_b,
12923                                   hw_plane_wm->trans_wm.plane_res_l);
12924                 }
12925
12926                 /* DDB */
12927                 hw_ddb_entry = &hw->ddb_y[plane];
12928                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
12929
12930                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12931                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12932                                   pipe_name(pipe), plane + 1,
12933                                   sw_ddb_entry->start, sw_ddb_entry->end,
12934                                   hw_ddb_entry->start, hw_ddb_entry->end);
12935                 }
12936         }
12937
12938         /*
12939          * cursor
12940          * If the cursor plane isn't active, we may not have updated it's ddb
12941          * allocation. In that case since the ddb allocation will be updated
12942          * once the plane becomes visible, we can skip this check
12943          */
12944         if (1) {
12945                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12946
12947                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
12948                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12949
12950                 /* Watermarks */
12951                 for (level = 0; level <= max_level; level++) {
12952                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12953                                                 &sw_plane_wm->wm[level]))
12954                                 continue;
12955
12956                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12957                                   pipe_name(pipe), level,
12958                                   sw_plane_wm->wm[level].plane_en,
12959                                   sw_plane_wm->wm[level].plane_res_b,
12960                                   sw_plane_wm->wm[level].plane_res_l,
12961                                   hw_plane_wm->wm[level].plane_en,
12962                                   hw_plane_wm->wm[level].plane_res_b,
12963                                   hw_plane_wm->wm[level].plane_res_l);
12964                 }
12965
12966                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12967                                          &sw_plane_wm->trans_wm)) {
12968                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12969                                   pipe_name(pipe),
12970                                   sw_plane_wm->trans_wm.plane_en,
12971                                   sw_plane_wm->trans_wm.plane_res_b,
12972                                   sw_plane_wm->trans_wm.plane_res_l,
12973                                   hw_plane_wm->trans_wm.plane_en,
12974                                   hw_plane_wm->trans_wm.plane_res_b,
12975                                   hw_plane_wm->trans_wm.plane_res_l);
12976                 }
12977
12978                 /* DDB */
12979                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
12980                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
12981
12982                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12983                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
12984                                   pipe_name(pipe),
12985                                   sw_ddb_entry->start, sw_ddb_entry->end,
12986                                   hw_ddb_entry->start, hw_ddb_entry->end);
12987                 }
12988         }
12989
12990         kfree(hw);
12991 }
12992
12993 static void
12994 verify_connector_state(struct intel_atomic_state *state,
12995                        struct intel_crtc *crtc)
12996 {
12997         struct drm_connector *connector;
12998         struct drm_connector_state *new_conn_state;
12999         int i;
13000
13001         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13002                 struct drm_encoder *encoder = connector->encoder;
13003                 struct intel_crtc_state *crtc_state = NULL;
13004
13005                 if (new_conn_state->crtc != &crtc->base)
13006                         continue;
13007
13008                 if (crtc)
13009                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13010
13011                 intel_connector_verify_state(crtc_state, new_conn_state);
13012
13013                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13014                      "connector's atomic encoder doesn't match legacy encoder\n");
13015         }
13016 }
13017
13018 static void
13019 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13020 {
13021         struct intel_encoder *encoder;
13022         struct drm_connector *connector;
13023         struct drm_connector_state *old_conn_state, *new_conn_state;
13024         int i;
13025
13026         for_each_intel_encoder(&dev_priv->drm, encoder) {
13027                 bool enabled = false, found = false;
13028                 enum pipe pipe;
13029
13030                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13031                               encoder->base.base.id,
13032                               encoder->base.name);
13033
13034                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13035                                                    new_conn_state, i) {
13036                         if (old_conn_state->best_encoder == &encoder->base)
13037                                 found = true;
13038
13039                         if (new_conn_state->best_encoder != &encoder->base)
13040                                 continue;
13041                         found = enabled = true;
13042
13043                         I915_STATE_WARN(new_conn_state->crtc !=
13044                                         encoder->base.crtc,
13045                              "connector's crtc doesn't match encoder crtc\n");
13046                 }
13047
13048                 if (!found)
13049                         continue;
13050
13051                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13052                      "encoder's enabled state mismatch "
13053                      "(expected %i, found %i)\n",
13054                      !!encoder->base.crtc, enabled);
13055
13056                 if (!encoder->base.crtc) {
13057                         bool active;
13058
13059                         active = encoder->get_hw_state(encoder, &pipe);
13060                         I915_STATE_WARN(active,
13061                              "encoder detached but still enabled on pipe %c.\n",
13062                              pipe_name(pipe));
13063                 }
13064         }
13065 }
13066
13067 static void
13068 verify_crtc_state(struct intel_crtc *crtc,
13069                   struct intel_crtc_state *old_crtc_state,
13070                   struct intel_crtc_state *new_crtc_state)
13071 {
13072         struct drm_device *dev = crtc->base.dev;
13073         struct drm_i915_private *dev_priv = to_i915(dev);
13074         struct intel_encoder *encoder;
13075         struct intel_crtc_state *pipe_config;
13076         struct drm_atomic_state *state;
13077         bool active;
13078
13079         state = old_crtc_state->base.state;
13080         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->base);
13081         pipe_config = old_crtc_state;
13082         memset(pipe_config, 0, sizeof(*pipe_config));
13083         pipe_config->base.crtc = &crtc->base;
13084         pipe_config->base.state = state;
13085
13086         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13087
13088         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13089
13090         /* we keep both pipes enabled on 830 */
13091         if (IS_I830(dev_priv))
13092                 active = new_crtc_state->base.active;
13093
13094         I915_STATE_WARN(new_crtc_state->base.active != active,
13095              "crtc active state doesn't match with hw state "
13096              "(expected %i, found %i)\n", new_crtc_state->base.active, active);
13097
13098         I915_STATE_WARN(crtc->active != new_crtc_state->base.active,
13099              "transitional active state does not match atomic hw state "
13100              "(expected %i, found %i)\n", new_crtc_state->base.active, crtc->active);
13101
13102         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13103                 enum pipe pipe;
13104
13105                 active = encoder->get_hw_state(encoder, &pipe);
13106                 I915_STATE_WARN(active != new_crtc_state->base.active,
13107                         "[ENCODER:%i] active %i with crtc active %i\n",
13108                         encoder->base.base.id, active, new_crtc_state->base.active);
13109
13110                 I915_STATE_WARN(active && crtc->pipe != pipe,
13111                                 "Encoder connected to wrong pipe %c\n",
13112                                 pipe_name(pipe));
13113
13114                 if (active)
13115                         encoder->get_config(encoder, pipe_config);
13116         }
13117
13118         intel_crtc_compute_pixel_rate(pipe_config);
13119
13120         if (!new_crtc_state->base.active)
13121                 return;
13122
13123         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13124
13125         if (!intel_pipe_config_compare(new_crtc_state,
13126                                        pipe_config, false)) {
13127                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13128                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13129                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13130         }
13131 }
13132
13133 static void
13134 intel_verify_planes(struct intel_atomic_state *state)
13135 {
13136         struct intel_plane *plane;
13137         const struct intel_plane_state *plane_state;
13138         int i;
13139
13140         for_each_new_intel_plane_in_state(state, plane,
13141                                           plane_state, i)
13142                 assert_plane(plane, plane_state->slave ||
13143                              plane_state->base.visible);
13144 }
13145
13146 static void
13147 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13148                          struct intel_shared_dpll *pll,
13149                          struct intel_crtc *crtc,
13150                          struct intel_crtc_state *new_crtc_state)
13151 {
13152         struct intel_dpll_hw_state dpll_hw_state;
13153         unsigned int crtc_mask;
13154         bool active;
13155
13156         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13157
13158         DRM_DEBUG_KMS("%s\n", pll->info->name);
13159
13160         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13161
13162         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13163                 I915_STATE_WARN(!pll->on && pll->active_mask,
13164                      "pll in active use but not on in sw tracking\n");
13165                 I915_STATE_WARN(pll->on && !pll->active_mask,
13166                      "pll is on but not used by any active crtc\n");
13167                 I915_STATE_WARN(pll->on != active,
13168                      "pll on state mismatch (expected %i, found %i)\n",
13169                      pll->on, active);
13170         }
13171
13172         if (!crtc) {
13173                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13174                                 "more active pll users than references: %x vs %x\n",
13175                                 pll->active_mask, pll->state.crtc_mask);
13176
13177                 return;
13178         }
13179
13180         crtc_mask = drm_crtc_mask(&crtc->base);
13181
13182         if (new_crtc_state->base.active)
13183                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13184                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13185                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13186         else
13187                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13188                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13189                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13190
13191         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13192                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13193                         crtc_mask, pll->state.crtc_mask);
13194
13195         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13196                                           &dpll_hw_state,
13197                                           sizeof(dpll_hw_state)),
13198                         "pll hw state mismatch\n");
13199 }
13200
13201 static void
13202 verify_shared_dpll_state(struct intel_crtc *crtc,
13203                          struct intel_crtc_state *old_crtc_state,
13204                          struct intel_crtc_state *new_crtc_state)
13205 {
13206         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13207
13208         if (new_crtc_state->shared_dpll)
13209                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13210
13211         if (old_crtc_state->shared_dpll &&
13212             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13213                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13214                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13215
13216                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13217                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13218                                 pipe_name(drm_crtc_index(&crtc->base)));
13219                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13220                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13221                                 pipe_name(drm_crtc_index(&crtc->base)));
13222         }
13223 }
13224
13225 static void
13226 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13227                           struct intel_atomic_state *state,
13228                           struct intel_crtc_state *old_crtc_state,
13229                           struct intel_crtc_state *new_crtc_state)
13230 {
13231         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13232                 return;
13233
13234         verify_wm_state(crtc, new_crtc_state);
13235         verify_connector_state(state, crtc);
13236         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13237         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13238 }
13239
13240 static void
13241 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13242 {
13243         int i;
13244
13245         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13246                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13247 }
13248
13249 static void
13250 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13251                               struct intel_atomic_state *state)
13252 {
13253         verify_encoder_state(dev_priv, state);
13254         verify_connector_state(state, NULL);
13255         verify_disabled_dpll_state(dev_priv);
13256 }
13257
13258 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
13259 {
13260         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13261         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13262
13263         /*
13264          * The scanline counter increments at the leading edge of hsync.
13265          *
13266          * On most platforms it starts counting from vtotal-1 on the
13267          * first active line. That means the scanline counter value is
13268          * always one less than what we would expect. Ie. just after
13269          * start of vblank, which also occurs at start of hsync (on the
13270          * last active line), the scanline counter will read vblank_start-1.
13271          *
13272          * On gen2 the scanline counter starts counting from 1 instead
13273          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13274          * to keep the value positive), instead of adding one.
13275          *
13276          * On HSW+ the behaviour of the scanline counter depends on the output
13277          * type. For DP ports it behaves like most other platforms, but on HDMI
13278          * there's an extra 1 line difference. So we need to add two instead of
13279          * one to the value.
13280          *
13281          * On VLV/CHV DSI the scanline counter would appear to increment
13282          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13283          * that means we can't tell whether we're in vblank or not while
13284          * we're on that particular line. We must still set scanline_offset
13285          * to 1 so that the vblank timestamps come out correct when we query
13286          * the scanline counter from within the vblank interrupt handler.
13287          * However if queried just before the start of vblank we'll get an
13288          * answer that's slightly in the future.
13289          */
13290         if (IS_GEN(dev_priv, 2)) {
13291                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
13292                 int vtotal;
13293
13294                 vtotal = adjusted_mode->crtc_vtotal;
13295                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13296                         vtotal /= 2;
13297
13298                 crtc->scanline_offset = vtotal - 1;
13299         } else if (HAS_DDI(dev_priv) &&
13300                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
13301                 crtc->scanline_offset = 2;
13302         } else
13303                 crtc->scanline_offset = 1;
13304 }
13305
13306 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
13307 {
13308         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13309         struct intel_crtc_state *new_crtc_state;
13310         struct intel_crtc *crtc;
13311         int i;
13312
13313         if (!dev_priv->display.crtc_compute_clock)
13314                 return;
13315
13316         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13317                 if (!needs_modeset(new_crtc_state))
13318                         continue;
13319
13320                 intel_release_shared_dplls(state, crtc);
13321         }
13322 }
13323
13324 /*
13325  * This implements the workaround described in the "notes" section of the mode
13326  * set sequence documentation. When going from no pipes or single pipe to
13327  * multiple pipes, and planes are enabled after the pipe, we need to wait at
13328  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
13329  */
13330 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
13331 {
13332         struct intel_crtc_state *crtc_state;
13333         struct intel_crtc *crtc;
13334         struct intel_crtc_state *first_crtc_state = NULL;
13335         struct intel_crtc_state *other_crtc_state = NULL;
13336         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13337         int i;
13338
13339         /* look at all crtc's that are going to be enabled in during modeset */
13340         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
13341                 if (!crtc_state->base.active ||
13342                     !needs_modeset(crtc_state))
13343                         continue;
13344
13345                 if (first_crtc_state) {
13346                         other_crtc_state = crtc_state;
13347                         break;
13348                 } else {
13349                         first_crtc_state = crtc_state;
13350                         first_pipe = crtc->pipe;
13351                 }
13352         }
13353
13354         /* No workaround needed? */
13355         if (!first_crtc_state)
13356                 return 0;
13357
13358         /* w/a possibly needed, check how many crtc's are already enabled. */
13359         for_each_intel_crtc(state->base.dev, crtc) {
13360                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13361                 if (IS_ERR(crtc_state))
13362                         return PTR_ERR(crtc_state);
13363
13364                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
13365
13366                 if (!crtc_state->base.active ||
13367                     needs_modeset(crtc_state))
13368                         continue;
13369
13370                 /* 2 or more enabled crtcs means no need for w/a */
13371                 if (enabled_pipe != INVALID_PIPE)
13372                         return 0;
13373
13374                 enabled_pipe = crtc->pipe;
13375         }
13376
13377         if (enabled_pipe != INVALID_PIPE)
13378                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13379         else if (other_crtc_state)
13380                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13381
13382         return 0;
13383 }
13384
13385 static int intel_lock_all_pipes(struct intel_atomic_state *state)
13386 {
13387         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13388         struct intel_crtc *crtc;
13389
13390         /* Add all pipes to the state */
13391         for_each_intel_crtc(&dev_priv->drm, crtc) {
13392                 struct intel_crtc_state *crtc_state;
13393
13394                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13395                 if (IS_ERR(crtc_state))
13396                         return PTR_ERR(crtc_state);
13397         }
13398
13399         return 0;
13400 }
13401
13402 static int intel_modeset_all_pipes(struct intel_atomic_state *state)
13403 {
13404         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13405         struct intel_crtc *crtc;
13406
13407         /*
13408          * Add all pipes to the state, and force
13409          * a modeset on all the active ones.
13410          */
13411         for_each_intel_crtc(&dev_priv->drm, crtc) {
13412                 struct intel_crtc_state *crtc_state;
13413                 int ret;
13414
13415                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13416                 if (IS_ERR(crtc_state))
13417                         return PTR_ERR(crtc_state);
13418
13419                 if (!crtc_state->base.active || needs_modeset(crtc_state))
13420                         continue;
13421
13422                 crtc_state->base.mode_changed = true;
13423
13424                 ret = drm_atomic_add_affected_connectors(&state->base,
13425                                                          &crtc->base);
13426                 if (ret)
13427                         return ret;
13428
13429                 ret = drm_atomic_add_affected_planes(&state->base,
13430                                                      &crtc->base);
13431                 if (ret)
13432                         return ret;
13433         }
13434
13435         return 0;
13436 }
13437
13438 static int intel_modeset_checks(struct intel_atomic_state *state)
13439 {
13440         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13441         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13442         struct intel_crtc *crtc;
13443         int ret = 0, i;
13444
13445         if (!check_digital_port_conflicts(state)) {
13446                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13447                 return -EINVAL;
13448         }
13449
13450         /* keep the current setting */
13451         if (!state->cdclk.force_min_cdclk_changed)
13452                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
13453
13454         state->modeset = true;
13455         state->active_pipes = dev_priv->active_pipes;
13456         state->cdclk.logical = dev_priv->cdclk.logical;
13457         state->cdclk.actual = dev_priv->cdclk.actual;
13458         state->cdclk.pipe = INVALID_PIPE;
13459
13460         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13461                                             new_crtc_state, i) {
13462                 if (new_crtc_state->base.active)
13463                         state->active_pipes |= BIT(crtc->pipe);
13464                 else
13465                         state->active_pipes &= ~BIT(crtc->pipe);
13466
13467                 if (old_crtc_state->base.active != new_crtc_state->base.active)
13468                         state->active_pipe_changes |= BIT(crtc->pipe);
13469         }
13470
13471         /*
13472          * See if the config requires any additional preparation, e.g.
13473          * to adjust global state with pipes off.  We need to do this
13474          * here so we can get the modeset_pipe updated config for the new
13475          * mode set on this crtc.  For other crtcs we need to use the
13476          * adjusted_mode bits in the crtc directly.
13477          */
13478         if (dev_priv->display.modeset_calc_cdclk) {
13479                 enum pipe pipe;
13480
13481                 ret = dev_priv->display.modeset_calc_cdclk(state);
13482                 if (ret < 0)
13483                         return ret;
13484
13485                 /*
13486                  * Writes to dev_priv->cdclk.logical must protected by
13487                  * holding all the crtc locks, even if we don't end up
13488                  * touching the hardware
13489                  */
13490                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
13491                                         &state->cdclk.logical)) {
13492                         ret = intel_lock_all_pipes(state);
13493                         if (ret < 0)
13494                                 return ret;
13495                 }
13496
13497                 if (is_power_of_2(state->active_pipes)) {
13498                         struct intel_crtc *crtc;
13499                         struct intel_crtc_state *crtc_state;
13500
13501                         pipe = ilog2(state->active_pipes);
13502                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
13503                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13504                         if (crtc_state && needs_modeset(crtc_state))
13505                                 pipe = INVALID_PIPE;
13506                 } else {
13507                         pipe = INVALID_PIPE;
13508                 }
13509
13510                 /* All pipes must be switched off while we change the cdclk. */
13511                 if (pipe != INVALID_PIPE &&
13512                     intel_cdclk_needs_cd2x_update(dev_priv,
13513                                                   &dev_priv->cdclk.actual,
13514                                                   &state->cdclk.actual)) {
13515                         ret = intel_lock_all_pipes(state);
13516                         if (ret < 0)
13517                                 return ret;
13518
13519                         state->cdclk.pipe = pipe;
13520                 } else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
13521                                                      &state->cdclk.actual)) {
13522                         ret = intel_modeset_all_pipes(state);
13523                         if (ret < 0)
13524                                 return ret;
13525
13526                         state->cdclk.pipe = INVALID_PIPE;
13527                 }
13528
13529                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
13530                               state->cdclk.logical.cdclk,
13531                               state->cdclk.actual.cdclk);
13532                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
13533                               state->cdclk.logical.voltage_level,
13534                               state->cdclk.actual.voltage_level);
13535         }
13536
13537         intel_modeset_clear_plls(state);
13538
13539         if (IS_HASWELL(dev_priv))
13540                 return haswell_mode_set_planes_workaround(state);
13541
13542         return 0;
13543 }
13544
13545 /*
13546  * Handle calculation of various watermark data at the end of the atomic check
13547  * phase.  The code here should be run after the per-crtc and per-plane 'check'
13548  * handlers to ensure that all derived state has been updated.
13549  */
13550 static int calc_watermark_data(struct intel_atomic_state *state)
13551 {
13552         struct drm_device *dev = state->base.dev;
13553         struct drm_i915_private *dev_priv = to_i915(dev);
13554
13555         /* Is there platform-specific watermark information to calculate? */
13556         if (dev_priv->display.compute_global_watermarks)
13557                 return dev_priv->display.compute_global_watermarks(state);
13558
13559         return 0;
13560 }
13561
13562 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
13563                                      struct intel_crtc_state *new_crtc_state)
13564 {
13565         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
13566                 return;
13567
13568         new_crtc_state->base.mode_changed = false;
13569         new_crtc_state->update_pipe = true;
13570
13571         /*
13572          * If we're not doing the full modeset we want to
13573          * keep the current M/N values as they may be
13574          * sufficiently different to the computed values
13575          * to cause problems.
13576          *
13577          * FIXME: should really copy more fuzzy state here
13578          */
13579         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
13580         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
13581         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
13582         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
13583 }
13584
13585 /**
13586  * intel_atomic_check - validate state object
13587  * @dev: drm device
13588  * @_state: state to validate
13589  */
13590 static int intel_atomic_check(struct drm_device *dev,
13591                               struct drm_atomic_state *_state)
13592 {
13593         struct drm_i915_private *dev_priv = to_i915(dev);
13594         struct intel_atomic_state *state = to_intel_atomic_state(_state);
13595         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13596         struct intel_crtc *crtc;
13597         int ret, i;
13598         bool any_ms = state->cdclk.force_min_cdclk_changed;
13599
13600         /* Catch I915_MODE_FLAG_INHERITED */
13601         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13602                                             new_crtc_state, i) {
13603                 if (new_crtc_state->base.mode.private_flags !=
13604                     old_crtc_state->base.mode.private_flags)
13605                         new_crtc_state->base.mode_changed = true;
13606         }
13607
13608         ret = drm_atomic_helper_check_modeset(dev, &state->base);
13609         if (ret)
13610                 goto fail;
13611
13612         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13613                                             new_crtc_state, i) {
13614                 if (!needs_modeset(new_crtc_state))
13615                         continue;
13616
13617                 if (!new_crtc_state->base.enable) {
13618                         any_ms = true;
13619                         continue;
13620                 }
13621
13622                 ret = intel_modeset_pipe_config(new_crtc_state);
13623                 if (ret)
13624                         goto fail;
13625
13626                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
13627
13628                 if (needs_modeset(new_crtc_state))
13629                         any_ms = true;
13630         }
13631
13632         ret = drm_dp_mst_atomic_check(&state->base);
13633         if (ret)
13634                 goto fail;
13635
13636         if (any_ms) {
13637                 ret = intel_modeset_checks(state);
13638                 if (ret)
13639                         goto fail;
13640         } else {
13641                 state->cdclk.logical = dev_priv->cdclk.logical;
13642         }
13643
13644         ret = icl_add_linked_planes(state);
13645         if (ret)
13646                 goto fail;
13647
13648         ret = drm_atomic_helper_check_planes(dev, &state->base);
13649         if (ret)
13650                 goto fail;
13651
13652         intel_fbc_choose_crtc(dev_priv, state);
13653         ret = calc_watermark_data(state);
13654         if (ret)
13655                 goto fail;
13656
13657         ret = intel_bw_atomic_check(state);
13658         if (ret)
13659                 goto fail;
13660
13661         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13662                                             new_crtc_state, i) {
13663                 if (!needs_modeset(new_crtc_state) &&
13664                     !new_crtc_state->update_pipe)
13665                         continue;
13666
13667                 intel_dump_pipe_config(new_crtc_state, state,
13668                                        needs_modeset(new_crtc_state) ?
13669                                        "[modeset]" : "[fastset]");
13670         }
13671
13672         return 0;
13673
13674  fail:
13675         if (ret == -EDEADLK)
13676                 return ret;
13677
13678         /*
13679          * FIXME would probably be nice to know which crtc specifically
13680          * caused the failure, in cases where we can pinpoint it.
13681          */
13682         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13683                                             new_crtc_state, i)
13684                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
13685
13686         return ret;
13687 }
13688
13689 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
13690 {
13691         return drm_atomic_helper_prepare_planes(state->base.dev,
13692                                                 &state->base);
13693 }
13694
13695 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
13696 {
13697         struct drm_device *dev = crtc->base.dev;
13698         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
13699
13700         if (!vblank->max_vblank_count)
13701                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
13702
13703         return crtc->base.funcs->get_vblank_counter(&crtc->base);
13704 }
13705
13706 static void intel_update_crtc(struct intel_crtc *crtc,
13707                               struct intel_atomic_state *state,
13708                               struct intel_crtc_state *old_crtc_state,
13709                               struct intel_crtc_state *new_crtc_state)
13710 {
13711         struct drm_device *dev = state->base.dev;
13712         struct drm_i915_private *dev_priv = to_i915(dev);
13713         bool modeset = needs_modeset(new_crtc_state);
13714         struct intel_plane_state *new_plane_state =
13715                 intel_atomic_get_new_plane_state(state,
13716                                                  to_intel_plane(crtc->base.primary));
13717
13718         if (modeset) {
13719                 update_scanline_offset(new_crtc_state);
13720                 dev_priv->display.crtc_enable(new_crtc_state, state);
13721
13722                 /* vblanks work again, re-enable pipe CRC. */
13723                 intel_crtc_enable_pipe_crc(crtc);
13724         } else {
13725                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13726
13727                 if (new_crtc_state->update_pipe)
13728                         intel_encoders_update_pipe(crtc, new_crtc_state, state);
13729         }
13730
13731         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
13732                 intel_fbc_disable(crtc);
13733         else if (new_plane_state)
13734                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
13735
13736         intel_begin_crtc_commit(state, crtc);
13737
13738         if (INTEL_GEN(dev_priv) >= 9)
13739                 skl_update_planes_on_crtc(state, crtc);
13740         else
13741                 i9xx_update_planes_on_crtc(state, crtc);
13742
13743         intel_finish_crtc_commit(state, crtc);
13744 }
13745
13746 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
13747 {
13748         struct intel_crtc *crtc;
13749         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13750         int i;
13751
13752         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13753                 if (!new_crtc_state->base.active)
13754                         continue;
13755
13756                 intel_update_crtc(crtc, state, old_crtc_state,
13757                                   new_crtc_state);
13758         }
13759 }
13760
13761 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
13762 {
13763         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13764         struct intel_crtc *crtc;
13765         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13766         unsigned int updated = 0;
13767         bool progress;
13768         enum pipe pipe;
13769         int i;
13770         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
13771         u8 required_slices = state->wm_results.ddb.enabled_slices;
13772         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
13773
13774         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
13775                 /* ignore allocations for crtc's that have been turned off. */
13776                 if (new_crtc_state->base.active)
13777                         entries[i] = old_crtc_state->wm.skl.ddb;
13778
13779         /* If 2nd DBuf slice required, enable it here */
13780         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
13781                 icl_dbuf_slices_update(dev_priv, required_slices);
13782
13783         /*
13784          * Whenever the number of active pipes changes, we need to make sure we
13785          * update the pipes in the right order so that their ddb allocations
13786          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
13787          * cause pipe underruns and other bad stuff.
13788          */
13789         do {
13790                 progress = false;
13791
13792                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13793                         bool vbl_wait = false;
13794                         unsigned int cmask = drm_crtc_mask(&crtc->base);
13795
13796                         pipe = crtc->pipe;
13797
13798                         if (updated & cmask || !new_crtc_state->base.active)
13799                                 continue;
13800
13801                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13802                                                         entries,
13803                                                         INTEL_INFO(dev_priv)->num_pipes, i))
13804                                 continue;
13805
13806                         updated |= cmask;
13807                         entries[i] = new_crtc_state->wm.skl.ddb;
13808
13809                         /*
13810                          * If this is an already active pipe, it's DDB changed,
13811                          * and this isn't the last pipe that needs updating
13812                          * then we need to wait for a vblank to pass for the
13813                          * new ddb allocation to take effect.
13814                          */
13815                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
13816                                                  &old_crtc_state->wm.skl.ddb) &&
13817                             !new_crtc_state->base.active_changed &&
13818                             state->wm_results.dirty_pipes != updated)
13819                                 vbl_wait = true;
13820
13821                         intel_update_crtc(crtc, state, old_crtc_state,
13822                                           new_crtc_state);
13823
13824                         if (vbl_wait)
13825                                 intel_wait_for_vblank(dev_priv, pipe);
13826
13827                         progress = true;
13828                 }
13829         } while (progress);
13830
13831         /* If 2nd DBuf slice is no more required disable it */
13832         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
13833                 icl_dbuf_slices_update(dev_priv, required_slices);
13834 }
13835
13836 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
13837 {
13838         struct intel_atomic_state *state, *next;
13839         struct llist_node *freed;
13840
13841         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
13842         llist_for_each_entry_safe(state, next, freed, freed)
13843                 drm_atomic_state_put(&state->base);
13844 }
13845
13846 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
13847 {
13848         struct drm_i915_private *dev_priv =
13849                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
13850
13851         intel_atomic_helper_free_state(dev_priv);
13852 }
13853
13854 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
13855 {
13856         struct wait_queue_entry wait_fence, wait_reset;
13857         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
13858
13859         init_wait_entry(&wait_fence, 0);
13860         init_wait_entry(&wait_reset, 0);
13861         for (;;) {
13862                 prepare_to_wait(&intel_state->commit_ready.wait,
13863                                 &wait_fence, TASK_UNINTERRUPTIBLE);
13864                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13865                                               I915_RESET_MODESET),
13866                                 &wait_reset, TASK_UNINTERRUPTIBLE);
13867
13868
13869                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
13870                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
13871                         break;
13872
13873                 schedule();
13874         }
13875         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
13876         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13877                                   I915_RESET_MODESET),
13878                     &wait_reset);
13879 }
13880
13881 static void intel_atomic_cleanup_work(struct work_struct *work)
13882 {
13883         struct drm_atomic_state *state =
13884                 container_of(work, struct drm_atomic_state, commit_work);
13885         struct drm_i915_private *i915 = to_i915(state->dev);
13886
13887         drm_atomic_helper_cleanup_planes(&i915->drm, state);
13888         drm_atomic_helper_commit_cleanup_done(state);
13889         drm_atomic_state_put(state);
13890
13891         intel_atomic_helper_free_state(i915);
13892 }
13893
13894 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
13895 {
13896         struct drm_device *dev = state->base.dev;
13897         struct drm_i915_private *dev_priv = to_i915(dev);
13898         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
13899         struct intel_crtc *crtc;
13900         u64 put_domains[I915_MAX_PIPES] = {};
13901         intel_wakeref_t wakeref = 0;
13902         int i;
13903
13904         intel_atomic_commit_fence_wait(state);
13905
13906         drm_atomic_helper_wait_for_dependencies(&state->base);
13907
13908         if (state->modeset)
13909                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13910
13911         /*
13912          * Disable CRTC/pipes in reverse order because some features(MST in
13913          * TGL+) requires master and slave relationship between pipes, so it
13914          * should always pick the lowest pipe as master as it will be enabled
13915          * first and disable in the reverse order so the master will be the
13916          * last one to be disabled.
13917          */
13918         for_each_oldnew_intel_crtc_in_state_reverse(state, crtc, old_crtc_state,
13919                                                     new_crtc_state, i) {
13920                 if (needs_modeset(new_crtc_state) ||
13921                     new_crtc_state->update_pipe) {
13922
13923                         put_domains[crtc->pipe] =
13924                                 modeset_get_crtc_power_domains(new_crtc_state);
13925                 }
13926
13927                 if (!needs_modeset(new_crtc_state))
13928                         continue;
13929
13930                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13931
13932                 if (old_crtc_state->base.active) {
13933                         intel_crtc_disable_planes(state, crtc);
13934
13935                         /*
13936                          * We need to disable pipe CRC before disabling the pipe,
13937                          * or we race against vblank off.
13938                          */
13939                         intel_crtc_disable_pipe_crc(crtc);
13940
13941                         dev_priv->display.crtc_disable(old_crtc_state, state);
13942                         crtc->active = false;
13943                         intel_fbc_disable(crtc);
13944                         intel_disable_shared_dpll(old_crtc_state);
13945
13946                         /*
13947                          * Underruns don't always raise
13948                          * interrupts, so check manually.
13949                          */
13950                         intel_check_cpu_fifo_underruns(dev_priv);
13951                         intel_check_pch_fifo_underruns(dev_priv);
13952
13953                         /* FIXME unify this for all platforms */
13954                         if (!new_crtc_state->base.active &&
13955                             !HAS_GMCH(dev_priv) &&
13956                             dev_priv->display.initial_watermarks)
13957                                 dev_priv->display.initial_watermarks(state,
13958                                                                      new_crtc_state);
13959                 }
13960         }
13961
13962         /* FIXME: Eventually get rid of our crtc->config pointer */
13963         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13964                 crtc->config = new_crtc_state;
13965
13966         if (state->modeset) {
13967                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
13968
13969                 intel_set_cdclk_pre_plane_update(dev_priv,
13970                                                  &state->cdclk.actual,
13971                                                  &dev_priv->cdclk.actual,
13972                                                  state->cdclk.pipe);
13973
13974                 /*
13975                  * SKL workaround: bspec recommends we disable the SAGV when we
13976                  * have more then one pipe enabled
13977                  */
13978                 if (!intel_can_enable_sagv(state))
13979                         intel_disable_sagv(dev_priv);
13980
13981                 intel_modeset_verify_disabled(dev_priv, state);
13982         }
13983
13984         /* Complete the events for pipes that have now been disabled */
13985         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13986                 bool modeset = needs_modeset(new_crtc_state);
13987
13988                 /* Complete events for now disable pipes here. */
13989                 if (modeset && !new_crtc_state->base.active && new_crtc_state->base.event) {
13990                         spin_lock_irq(&dev->event_lock);
13991                         drm_crtc_send_vblank_event(&crtc->base, new_crtc_state->base.event);
13992                         spin_unlock_irq(&dev->event_lock);
13993
13994                         new_crtc_state->base.event = NULL;
13995                 }
13996         }
13997
13998         if (state->modeset)
13999                 intel_encoders_update_prepare(state);
14000
14001         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
14002         dev_priv->display.commit_modeset_enables(state);
14003
14004         if (state->modeset) {
14005                 intel_encoders_update_complete(state);
14006
14007                 intel_set_cdclk_post_plane_update(dev_priv,
14008                                                   &state->cdclk.actual,
14009                                                   &dev_priv->cdclk.actual,
14010                                                   state->cdclk.pipe);
14011         }
14012
14013         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
14014          * already, but still need the state for the delayed optimization. To
14015          * fix this:
14016          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
14017          * - schedule that vblank worker _before_ calling hw_done
14018          * - at the start of commit_tail, cancel it _synchrously
14019          * - switch over to the vblank wait helper in the core after that since
14020          *   we don't need out special handling any more.
14021          */
14022         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
14023
14024         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14025                 if (new_crtc_state->base.active &&
14026                     !needs_modeset(new_crtc_state) &&
14027                     (new_crtc_state->base.color_mgmt_changed ||
14028                      new_crtc_state->update_pipe))
14029                         intel_color_load_luts(new_crtc_state);
14030         }
14031
14032         /*
14033          * Now that the vblank has passed, we can go ahead and program the
14034          * optimal watermarks on platforms that need two-step watermark
14035          * programming.
14036          *
14037          * TODO: Move this (and other cleanup) to an async worker eventually.
14038          */
14039         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14040                 if (dev_priv->display.optimize_watermarks)
14041                         dev_priv->display.optimize_watermarks(state,
14042                                                               new_crtc_state);
14043         }
14044
14045         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14046                 intel_post_plane_update(old_crtc_state);
14047
14048                 if (put_domains[i])
14049                         modeset_put_power_domains(dev_priv, put_domains[i]);
14050
14051                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
14052         }
14053
14054         if (state->modeset)
14055                 intel_verify_planes(state);
14056
14057         if (state->modeset && intel_can_enable_sagv(state))
14058                 intel_enable_sagv(dev_priv);
14059
14060         drm_atomic_helper_commit_hw_done(&state->base);
14061
14062         if (state->modeset) {
14063                 /* As one of the primary mmio accessors, KMS has a high
14064                  * likelihood of triggering bugs in unclaimed access. After we
14065                  * finish modesetting, see if an error has been flagged, and if
14066                  * so enable debugging for the next modeset - and hope we catch
14067                  * the culprit.
14068                  */
14069                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
14070                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
14071         }
14072         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14073
14074         /*
14075          * Defer the cleanup of the old state to a separate worker to not
14076          * impede the current task (userspace for blocking modesets) that
14077          * are executed inline. For out-of-line asynchronous modesets/flips,
14078          * deferring to a new worker seems overkill, but we would place a
14079          * schedule point (cond_resched()) here anyway to keep latencies
14080          * down.
14081          */
14082         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
14083         queue_work(system_highpri_wq, &state->base.commit_work);
14084 }
14085
14086 static void intel_atomic_commit_work(struct work_struct *work)
14087 {
14088         struct intel_atomic_state *state =
14089                 container_of(work, struct intel_atomic_state, base.commit_work);
14090
14091         intel_atomic_commit_tail(state);
14092 }
14093
14094 static int __i915_sw_fence_call
14095 intel_atomic_commit_ready(struct i915_sw_fence *fence,
14096                           enum i915_sw_fence_notify notify)
14097 {
14098         struct intel_atomic_state *state =
14099                 container_of(fence, struct intel_atomic_state, commit_ready);
14100
14101         switch (notify) {
14102         case FENCE_COMPLETE:
14103                 /* we do blocking waits in the worker, nothing to do here */
14104                 break;
14105         case FENCE_FREE:
14106                 {
14107                         struct intel_atomic_helper *helper =
14108                                 &to_i915(state->base.dev)->atomic_helper;
14109
14110                         if (llist_add(&state->freed, &helper->free_list))
14111                                 schedule_work(&helper->free_work);
14112                         break;
14113                 }
14114         }
14115
14116         return NOTIFY_DONE;
14117 }
14118
14119 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
14120 {
14121         struct intel_plane_state *old_plane_state, *new_plane_state;
14122         struct intel_plane *plane;
14123         int i;
14124
14125         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
14126                                              new_plane_state, i)
14127                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->base.fb),
14128                                         to_intel_frontbuffer(new_plane_state->base.fb),
14129                                         plane->frontbuffer_bit);
14130 }
14131
14132 static int intel_atomic_commit(struct drm_device *dev,
14133                                struct drm_atomic_state *_state,
14134                                bool nonblock)
14135 {
14136         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14137         struct drm_i915_private *dev_priv = to_i915(dev);
14138         int ret = 0;
14139
14140         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
14141
14142         drm_atomic_state_get(&state->base);
14143         i915_sw_fence_init(&state->commit_ready,
14144                            intel_atomic_commit_ready);
14145
14146         /*
14147          * The intel_legacy_cursor_update() fast path takes care
14148          * of avoiding the vblank waits for simple cursor
14149          * movement and flips. For cursor on/off and size changes,
14150          * we want to perform the vblank waits so that watermark
14151          * updates happen during the correct frames. Gen9+ have
14152          * double buffered watermarks and so shouldn't need this.
14153          *
14154          * Unset state->legacy_cursor_update before the call to
14155          * drm_atomic_helper_setup_commit() because otherwise
14156          * drm_atomic_helper_wait_for_flip_done() is a noop and
14157          * we get FIFO underruns because we didn't wait
14158          * for vblank.
14159          *
14160          * FIXME doing watermarks and fb cleanup from a vblank worker
14161          * (assuming we had any) would solve these problems.
14162          */
14163         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
14164                 struct intel_crtc_state *new_crtc_state;
14165                 struct intel_crtc *crtc;
14166                 int i;
14167
14168                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
14169                         if (new_crtc_state->wm.need_postvbl_update ||
14170                             new_crtc_state->update_wm_post)
14171                                 state->base.legacy_cursor_update = false;
14172         }
14173
14174         ret = intel_atomic_prepare_commit(state);
14175         if (ret) {
14176                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
14177                 i915_sw_fence_commit(&state->commit_ready);
14178                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14179                 return ret;
14180         }
14181
14182         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
14183         if (!ret)
14184                 ret = drm_atomic_helper_swap_state(&state->base, true);
14185
14186         if (ret) {
14187                 i915_sw_fence_commit(&state->commit_ready);
14188
14189                 drm_atomic_helper_cleanup_planes(dev, &state->base);
14190                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14191                 return ret;
14192         }
14193         dev_priv->wm.distrust_bios_wm = false;
14194         intel_shared_dpll_swap_state(state);
14195         intel_atomic_track_fbs(state);
14196
14197         if (state->modeset) {
14198                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
14199                        sizeof(state->min_cdclk));
14200                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
14201                        sizeof(state->min_voltage_level));
14202                 dev_priv->active_pipes = state->active_pipes;
14203                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
14204
14205                 intel_cdclk_swap_state(state);
14206         }
14207
14208         drm_atomic_state_get(&state->base);
14209         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
14210
14211         i915_sw_fence_commit(&state->commit_ready);
14212         if (nonblock && state->modeset) {
14213                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
14214         } else if (nonblock) {
14215                 queue_work(system_unbound_wq, &state->base.commit_work);
14216         } else {
14217                 if (state->modeset)
14218                         flush_workqueue(dev_priv->modeset_wq);
14219                 intel_atomic_commit_tail(state);
14220         }
14221
14222         return 0;
14223 }
14224
14225 struct wait_rps_boost {
14226         struct wait_queue_entry wait;
14227
14228         struct drm_crtc *crtc;
14229         struct i915_request *request;
14230 };
14231
14232 static int do_rps_boost(struct wait_queue_entry *_wait,
14233                         unsigned mode, int sync, void *key)
14234 {
14235         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
14236         struct i915_request *rq = wait->request;
14237
14238         /*
14239          * If we missed the vblank, but the request is already running it
14240          * is reasonable to assume that it will complete before the next
14241          * vblank without our intervention, so leave RPS alone.
14242          */
14243         if (!i915_request_started(rq))
14244                 gen6_rps_boost(rq);
14245         i915_request_put(rq);
14246
14247         drm_crtc_vblank_put(wait->crtc);
14248
14249         list_del(&wait->wait.entry);
14250         kfree(wait);
14251         return 1;
14252 }
14253
14254 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
14255                                        struct dma_fence *fence)
14256 {
14257         struct wait_rps_boost *wait;
14258
14259         if (!dma_fence_is_i915(fence))
14260                 return;
14261
14262         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
14263                 return;
14264
14265         if (drm_crtc_vblank_get(crtc))
14266                 return;
14267
14268         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
14269         if (!wait) {
14270                 drm_crtc_vblank_put(crtc);
14271                 return;
14272         }
14273
14274         wait->request = to_request(dma_fence_get(fence));
14275         wait->crtc = crtc;
14276
14277         wait->wait.func = do_rps_boost;
14278         wait->wait.flags = 0;
14279
14280         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
14281 }
14282
14283 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
14284 {
14285         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
14286         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
14287         struct drm_framebuffer *fb = plane_state->base.fb;
14288         struct i915_vma *vma;
14289
14290         if (plane->id == PLANE_CURSOR &&
14291             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
14292                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14293                 const int align = intel_cursor_alignment(dev_priv);
14294                 int err;
14295
14296                 err = i915_gem_object_attach_phys(obj, align);
14297                 if (err)
14298                         return err;
14299         }
14300
14301         vma = intel_pin_and_fence_fb_obj(fb,
14302                                          &plane_state->view,
14303                                          intel_plane_uses_fence(plane_state),
14304                                          &plane_state->flags);
14305         if (IS_ERR(vma))
14306                 return PTR_ERR(vma);
14307
14308         plane_state->vma = vma;
14309
14310         return 0;
14311 }
14312
14313 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
14314 {
14315         struct i915_vma *vma;
14316
14317         vma = fetch_and_zero(&old_plane_state->vma);
14318         if (vma)
14319                 intel_unpin_fb_vma(vma, old_plane_state->flags);
14320 }
14321
14322 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
14323 {
14324         struct i915_sched_attr attr = {
14325                 .priority = I915_PRIORITY_DISPLAY,
14326         };
14327
14328         i915_gem_object_wait_priority(obj, 0, &attr);
14329 }
14330
14331 /**
14332  * intel_prepare_plane_fb - Prepare fb for usage on plane
14333  * @plane: drm plane to prepare for
14334  * @new_state: the plane state being prepared
14335  *
14336  * Prepares a framebuffer for usage on a display plane.  Generally this
14337  * involves pinning the underlying object and updating the frontbuffer tracking
14338  * bits.  Some older platforms need special physical address handling for
14339  * cursor planes.
14340  *
14341  * Must be called with struct_mutex held.
14342  *
14343  * Returns 0 on success, negative error code on failure.
14344  */
14345 int
14346 intel_prepare_plane_fb(struct drm_plane *plane,
14347                        struct drm_plane_state *new_state)
14348 {
14349         struct intel_atomic_state *intel_state =
14350                 to_intel_atomic_state(new_state->state);
14351         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14352         struct drm_framebuffer *fb = new_state->fb;
14353         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14354         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
14355         int ret;
14356
14357         if (old_obj) {
14358                 struct intel_crtc_state *crtc_state =
14359                         intel_atomic_get_new_crtc_state(intel_state,
14360                                                         to_intel_crtc(plane->state->crtc));
14361
14362                 /* Big Hammer, we also need to ensure that any pending
14363                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
14364                  * current scanout is retired before unpinning the old
14365                  * framebuffer. Note that we rely on userspace rendering
14366                  * into the buffer attached to the pipe they are waiting
14367                  * on. If not, userspace generates a GPU hang with IPEHR
14368                  * point to the MI_WAIT_FOR_EVENT.
14369                  *
14370                  * This should only fail upon a hung GPU, in which case we
14371                  * can safely continue.
14372                  */
14373                 if (needs_modeset(crtc_state)) {
14374                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14375                                                               old_obj->base.resv, NULL,
14376                                                               false, 0,
14377                                                               GFP_KERNEL);
14378                         if (ret < 0)
14379                                 return ret;
14380                 }
14381         }
14382
14383         if (new_state->fence) { /* explicit fencing */
14384                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
14385                                                     new_state->fence,
14386                                                     I915_FENCE_TIMEOUT,
14387                                                     GFP_KERNEL);
14388                 if (ret < 0)
14389                         return ret;
14390         }
14391
14392         if (!obj)
14393                 return 0;
14394
14395         ret = i915_gem_object_pin_pages(obj);
14396         if (ret)
14397                 return ret;
14398
14399         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14400         if (ret) {
14401                 i915_gem_object_unpin_pages(obj);
14402                 return ret;
14403         }
14404
14405         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
14406
14407         mutex_unlock(&dev_priv->drm.struct_mutex);
14408         i915_gem_object_unpin_pages(obj);
14409         if (ret)
14410                 return ret;
14411
14412         fb_obj_bump_render_priority(obj);
14413         intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_DIRTYFB);
14414
14415         if (!new_state->fence) { /* implicit fencing */
14416                 struct dma_fence *fence;
14417
14418                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14419                                                       obj->base.resv, NULL,
14420                                                       false, I915_FENCE_TIMEOUT,
14421                                                       GFP_KERNEL);
14422                 if (ret < 0)
14423                         return ret;
14424
14425                 fence = dma_resv_get_excl_rcu(obj->base.resv);
14426                 if (fence) {
14427                         add_rps_boost_after_vblank(new_state->crtc, fence);
14428                         dma_fence_put(fence);
14429                 }
14430         } else {
14431                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
14432         }
14433
14434         /*
14435          * We declare pageflips to be interactive and so merit a small bias
14436          * towards upclocking to deliver the frame on time. By only changing
14437          * the RPS thresholds to sample more regularly and aim for higher
14438          * clocks we can hopefully deliver low power workloads (like kodi)
14439          * that are not quite steady state without resorting to forcing
14440          * maximum clocks following a vblank miss (see do_rps_boost()).
14441          */
14442         if (!intel_state->rps_interactive) {
14443                 intel_rps_mark_interactive(dev_priv, true);
14444                 intel_state->rps_interactive = true;
14445         }
14446
14447         return 0;
14448 }
14449
14450 /**
14451  * intel_cleanup_plane_fb - Cleans up an fb after plane use
14452  * @plane: drm plane to clean up for
14453  * @old_state: the state from the previous modeset
14454  *
14455  * Cleans up a framebuffer that has just been removed from a plane.
14456  *
14457  * Must be called with struct_mutex held.
14458  */
14459 void
14460 intel_cleanup_plane_fb(struct drm_plane *plane,
14461                        struct drm_plane_state *old_state)
14462 {
14463         struct intel_atomic_state *intel_state =
14464                 to_intel_atomic_state(old_state->state);
14465         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14466
14467         if (intel_state->rps_interactive) {
14468                 intel_rps_mark_interactive(dev_priv, false);
14469                 intel_state->rps_interactive = false;
14470         }
14471
14472         /* Should only be called after a successful intel_prepare_plane_fb()! */
14473         mutex_lock(&dev_priv->drm.struct_mutex);
14474         intel_plane_unpin_fb(to_intel_plane_state(old_state));
14475         mutex_unlock(&dev_priv->drm.struct_mutex);
14476 }
14477
14478 int
14479 skl_max_scale(const struct intel_crtc_state *crtc_state,
14480               u32 pixel_format)
14481 {
14482         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
14483         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14484         int max_scale, mult;
14485         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
14486
14487         if (!crtc_state->base.enable)
14488                 return DRM_PLANE_HELPER_NO_SCALING;
14489
14490         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
14491         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
14492
14493         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
14494                 max_dotclk *= 2;
14495
14496         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
14497                 return DRM_PLANE_HELPER_NO_SCALING;
14498
14499         /*
14500          * skl max scale is lower of:
14501          *    close to 3 but not 3, -1 is for that purpose
14502          *            or
14503          *    cdclk/crtc_clock
14504          */
14505         mult = is_planar_yuv_format(pixel_format) ? 2 : 3;
14506         tmpclk1 = (1 << 16) * mult - 1;
14507         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
14508         max_scale = min(tmpclk1, tmpclk2);
14509
14510         return max_scale;
14511 }
14512
14513 static void intel_begin_crtc_commit(struct intel_atomic_state *state,
14514                                     struct intel_crtc *crtc)
14515 {
14516         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14517         struct intel_crtc_state *old_crtc_state =
14518                 intel_atomic_get_old_crtc_state(state, crtc);
14519         struct intel_crtc_state *new_crtc_state =
14520                 intel_atomic_get_new_crtc_state(state, crtc);
14521         bool modeset = needs_modeset(new_crtc_state);
14522
14523         /* Perform vblank evasion around commit operation */
14524         intel_pipe_update_start(new_crtc_state);
14525
14526         if (modeset)
14527                 goto out;
14528
14529         if (new_crtc_state->base.color_mgmt_changed ||
14530             new_crtc_state->update_pipe)
14531                 intel_color_commit(new_crtc_state);
14532
14533         if (new_crtc_state->update_pipe)
14534                 intel_update_pipe_config(old_crtc_state, new_crtc_state);
14535         else if (INTEL_GEN(dev_priv) >= 9)
14536                 skl_detach_scalers(new_crtc_state);
14537
14538         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14539                 bdw_set_pipemisc(new_crtc_state);
14540
14541 out:
14542         if (dev_priv->display.atomic_update_watermarks)
14543                 dev_priv->display.atomic_update_watermarks(state,
14544                                                            new_crtc_state);
14545 }
14546
14547 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14548                                   struct intel_crtc_state *crtc_state)
14549 {
14550         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14551
14552         if (!IS_GEN(dev_priv, 2))
14553                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14554
14555         if (crtc_state->has_pch_encoder) {
14556                 enum pipe pch_transcoder =
14557                         intel_crtc_pch_transcoder(crtc);
14558
14559                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14560         }
14561 }
14562
14563 static void intel_finish_crtc_commit(struct intel_atomic_state *state,
14564                                      struct intel_crtc *crtc)
14565 {
14566         struct intel_crtc_state *old_crtc_state =
14567                 intel_atomic_get_old_crtc_state(state, crtc);
14568         struct intel_crtc_state *new_crtc_state =
14569                 intel_atomic_get_new_crtc_state(state, crtc);
14570
14571         intel_pipe_update_end(new_crtc_state);
14572
14573         if (new_crtc_state->update_pipe &&
14574             !needs_modeset(new_crtc_state) &&
14575             old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED)
14576                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14577 }
14578
14579 /**
14580  * intel_plane_destroy - destroy a plane
14581  * @plane: plane to destroy
14582  *
14583  * Common destruction function for all types of planes (primary, cursor,
14584  * sprite).
14585  */
14586 void intel_plane_destroy(struct drm_plane *plane)
14587 {
14588         drm_plane_cleanup(plane);
14589         kfree(to_intel_plane(plane));
14590 }
14591
14592 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
14593                                             u32 format, u64 modifier)
14594 {
14595         switch (modifier) {
14596         case DRM_FORMAT_MOD_LINEAR:
14597         case I915_FORMAT_MOD_X_TILED:
14598                 break;
14599         default:
14600                 return false;
14601         }
14602
14603         switch (format) {
14604         case DRM_FORMAT_C8:
14605         case DRM_FORMAT_RGB565:
14606         case DRM_FORMAT_XRGB1555:
14607         case DRM_FORMAT_XRGB8888:
14608                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14609                         modifier == I915_FORMAT_MOD_X_TILED;
14610         default:
14611                 return false;
14612         }
14613 }
14614
14615 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
14616                                             u32 format, u64 modifier)
14617 {
14618         switch (modifier) {
14619         case DRM_FORMAT_MOD_LINEAR:
14620         case I915_FORMAT_MOD_X_TILED:
14621                 break;
14622         default:
14623                 return false;
14624         }
14625
14626         switch (format) {
14627         case DRM_FORMAT_C8:
14628         case DRM_FORMAT_RGB565:
14629         case DRM_FORMAT_XRGB8888:
14630         case DRM_FORMAT_XBGR8888:
14631         case DRM_FORMAT_XRGB2101010:
14632         case DRM_FORMAT_XBGR2101010:
14633                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14634                         modifier == I915_FORMAT_MOD_X_TILED;
14635         default:
14636                 return false;
14637         }
14638 }
14639
14640 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
14641                                               u32 format, u64 modifier)
14642 {
14643         return modifier == DRM_FORMAT_MOD_LINEAR &&
14644                 format == DRM_FORMAT_ARGB8888;
14645 }
14646
14647 static const struct drm_plane_funcs i965_plane_funcs = {
14648         .update_plane = drm_atomic_helper_update_plane,
14649         .disable_plane = drm_atomic_helper_disable_plane,
14650         .destroy = intel_plane_destroy,
14651         .atomic_duplicate_state = intel_plane_duplicate_state,
14652         .atomic_destroy_state = intel_plane_destroy_state,
14653         .format_mod_supported = i965_plane_format_mod_supported,
14654 };
14655
14656 static const struct drm_plane_funcs i8xx_plane_funcs = {
14657         .update_plane = drm_atomic_helper_update_plane,
14658         .disable_plane = drm_atomic_helper_disable_plane,
14659         .destroy = intel_plane_destroy,
14660         .atomic_duplicate_state = intel_plane_duplicate_state,
14661         .atomic_destroy_state = intel_plane_destroy_state,
14662         .format_mod_supported = i8xx_plane_format_mod_supported,
14663 };
14664
14665 static int
14666 intel_legacy_cursor_update(struct drm_plane *plane,
14667                            struct drm_crtc *crtc,
14668                            struct drm_framebuffer *fb,
14669                            int crtc_x, int crtc_y,
14670                            unsigned int crtc_w, unsigned int crtc_h,
14671                            u32 src_x, u32 src_y,
14672                            u32 src_w, u32 src_h,
14673                            struct drm_modeset_acquire_ctx *ctx)
14674 {
14675         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
14676         struct drm_plane_state *old_plane_state, *new_plane_state;
14677         struct intel_plane *intel_plane = to_intel_plane(plane);
14678         struct intel_crtc_state *crtc_state =
14679                 to_intel_crtc_state(crtc->state);
14680         struct intel_crtc_state *new_crtc_state;
14681         int ret;
14682
14683         /*
14684          * When crtc is inactive or there is a modeset pending,
14685          * wait for it to complete in the slowpath
14686          */
14687         if (!crtc_state->base.active || needs_modeset(crtc_state) ||
14688             crtc_state->update_pipe)
14689                 goto slow;
14690
14691         old_plane_state = plane->state;
14692         /*
14693          * Don't do an async update if there is an outstanding commit modifying
14694          * the plane.  This prevents our async update's changes from getting
14695          * overridden by a previous synchronous update's state.
14696          */
14697         if (old_plane_state->commit &&
14698             !try_wait_for_completion(&old_plane_state->commit->hw_done))
14699                 goto slow;
14700
14701         /*
14702          * If any parameters change that may affect watermarks,
14703          * take the slowpath. Only changing fb or position should be
14704          * in the fastpath.
14705          */
14706         if (old_plane_state->crtc != crtc ||
14707             old_plane_state->src_w != src_w ||
14708             old_plane_state->src_h != src_h ||
14709             old_plane_state->crtc_w != crtc_w ||
14710             old_plane_state->crtc_h != crtc_h ||
14711             !old_plane_state->fb != !fb)
14712                 goto slow;
14713
14714         new_plane_state = intel_plane_duplicate_state(plane);
14715         if (!new_plane_state)
14716                 return -ENOMEM;
14717
14718         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
14719         if (!new_crtc_state) {
14720                 ret = -ENOMEM;
14721                 goto out_free;
14722         }
14723
14724         drm_atomic_set_fb_for_plane(new_plane_state, fb);
14725
14726         new_plane_state->src_x = src_x;
14727         new_plane_state->src_y = src_y;
14728         new_plane_state->src_w = src_w;
14729         new_plane_state->src_h = src_h;
14730         new_plane_state->crtc_x = crtc_x;
14731         new_plane_state->crtc_y = crtc_y;
14732         new_plane_state->crtc_w = crtc_w;
14733         new_plane_state->crtc_h = crtc_h;
14734
14735         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
14736                                                   to_intel_plane_state(old_plane_state),
14737                                                   to_intel_plane_state(new_plane_state));
14738         if (ret)
14739                 goto out_free;
14740
14741         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14742         if (ret)
14743                 goto out_free;
14744
14745         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
14746         if (ret)
14747                 goto out_unlock;
14748
14749         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_FLIP);
14750         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->fb),
14751                                 to_intel_frontbuffer(fb),
14752                                 intel_plane->frontbuffer_bit);
14753
14754         /* Swap plane state */
14755         plane->state = new_plane_state;
14756
14757         /*
14758          * We cannot swap crtc_state as it may be in use by an atomic commit or
14759          * page flip that's running simultaneously. If we swap crtc_state and
14760          * destroy the old state, we will cause a use-after-free there.
14761          *
14762          * Only update active_planes, which is needed for our internal
14763          * bookkeeping. Either value will do the right thing when updating
14764          * planes atomically. If the cursor was part of the atomic update then
14765          * we would have taken the slowpath.
14766          */
14767         crtc_state->active_planes = new_crtc_state->active_planes;
14768
14769         if (plane->state->visible)
14770                 intel_update_plane(intel_plane, crtc_state,
14771                                    to_intel_plane_state(plane->state));
14772         else
14773                 intel_disable_plane(intel_plane, crtc_state);
14774
14775         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
14776
14777 out_unlock:
14778         mutex_unlock(&dev_priv->drm.struct_mutex);
14779 out_free:
14780         if (new_crtc_state)
14781                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
14782         if (ret)
14783                 intel_plane_destroy_state(plane, new_plane_state);
14784         else
14785                 intel_plane_destroy_state(plane, old_plane_state);
14786         return ret;
14787
14788 slow:
14789         return drm_atomic_helper_update_plane(plane, crtc, fb,
14790                                               crtc_x, crtc_y, crtc_w, crtc_h,
14791                                               src_x, src_y, src_w, src_h, ctx);
14792 }
14793
14794 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
14795         .update_plane = intel_legacy_cursor_update,
14796         .disable_plane = drm_atomic_helper_disable_plane,
14797         .destroy = intel_plane_destroy,
14798         .atomic_duplicate_state = intel_plane_duplicate_state,
14799         .atomic_destroy_state = intel_plane_destroy_state,
14800         .format_mod_supported = intel_cursor_format_mod_supported,
14801 };
14802
14803 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
14804                                enum i9xx_plane_id i9xx_plane)
14805 {
14806         if (!HAS_FBC(dev_priv))
14807                 return false;
14808
14809         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
14810                 return i9xx_plane == PLANE_A; /* tied to pipe A */
14811         else if (IS_IVYBRIDGE(dev_priv))
14812                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
14813                         i9xx_plane == PLANE_C;
14814         else if (INTEL_GEN(dev_priv) >= 4)
14815                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
14816         else
14817                 return i9xx_plane == PLANE_A;
14818 }
14819
14820 static struct intel_plane *
14821 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
14822 {
14823         struct intel_plane *plane;
14824         const struct drm_plane_funcs *plane_funcs;
14825         unsigned int supported_rotations;
14826         unsigned int possible_crtcs;
14827         const u64 *modifiers;
14828         const u32 *formats;
14829         int num_formats;
14830         int ret;
14831
14832         if (INTEL_GEN(dev_priv) >= 9)
14833                 return skl_universal_plane_create(dev_priv, pipe,
14834                                                   PLANE_PRIMARY);
14835
14836         plane = intel_plane_alloc();
14837         if (IS_ERR(plane))
14838                 return plane;
14839
14840         plane->pipe = pipe;
14841         /*
14842          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
14843          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
14844          */
14845         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14846                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
14847         else
14848                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
14849         plane->id = PLANE_PRIMARY;
14850         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
14851
14852         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
14853         if (plane->has_fbc) {
14854                 struct intel_fbc *fbc = &dev_priv->fbc;
14855
14856                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
14857         }
14858
14859         if (INTEL_GEN(dev_priv) >= 4) {
14860                 formats = i965_primary_formats;
14861                 num_formats = ARRAY_SIZE(i965_primary_formats);
14862                 modifiers = i9xx_format_modifiers;
14863
14864                 plane->max_stride = i9xx_plane_max_stride;
14865                 plane->update_plane = i9xx_update_plane;
14866                 plane->disable_plane = i9xx_disable_plane;
14867                 plane->get_hw_state = i9xx_plane_get_hw_state;
14868                 plane->check_plane = i9xx_plane_check;
14869
14870                 plane_funcs = &i965_plane_funcs;
14871         } else {
14872                 formats = i8xx_primary_formats;
14873                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
14874                 modifiers = i9xx_format_modifiers;
14875
14876                 plane->max_stride = i9xx_plane_max_stride;
14877                 plane->update_plane = i9xx_update_plane;
14878                 plane->disable_plane = i9xx_disable_plane;
14879                 plane->get_hw_state = i9xx_plane_get_hw_state;
14880                 plane->check_plane = i9xx_plane_check;
14881
14882                 plane_funcs = &i8xx_plane_funcs;
14883         }
14884
14885         possible_crtcs = BIT(pipe);
14886
14887         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
14888                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14889                                                possible_crtcs, plane_funcs,
14890                                                formats, num_formats, modifiers,
14891                                                DRM_PLANE_TYPE_PRIMARY,
14892                                                "primary %c", pipe_name(pipe));
14893         else
14894                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14895                                                possible_crtcs, plane_funcs,
14896                                                formats, num_formats, modifiers,
14897                                                DRM_PLANE_TYPE_PRIMARY,
14898                                                "plane %c",
14899                                                plane_name(plane->i9xx_plane));
14900         if (ret)
14901                 goto fail;
14902
14903         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
14904                 supported_rotations =
14905                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
14906                         DRM_MODE_REFLECT_X;
14907         } else if (INTEL_GEN(dev_priv) >= 4) {
14908                 supported_rotations =
14909                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
14910         } else {
14911                 supported_rotations = DRM_MODE_ROTATE_0;
14912         }
14913
14914         if (INTEL_GEN(dev_priv) >= 4)
14915                 drm_plane_create_rotation_property(&plane->base,
14916                                                    DRM_MODE_ROTATE_0,
14917                                                    supported_rotations);
14918
14919         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
14920
14921         return plane;
14922
14923 fail:
14924         intel_plane_free(plane);
14925
14926         return ERR_PTR(ret);
14927 }
14928
14929 static struct intel_plane *
14930 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
14931                           enum pipe pipe)
14932 {
14933         unsigned int possible_crtcs;
14934         struct intel_plane *cursor;
14935         int ret;
14936
14937         cursor = intel_plane_alloc();
14938         if (IS_ERR(cursor))
14939                 return cursor;
14940
14941         cursor->pipe = pipe;
14942         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
14943         cursor->id = PLANE_CURSOR;
14944         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
14945
14946         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
14947                 cursor->max_stride = i845_cursor_max_stride;
14948                 cursor->update_plane = i845_update_cursor;
14949                 cursor->disable_plane = i845_disable_cursor;
14950                 cursor->get_hw_state = i845_cursor_get_hw_state;
14951                 cursor->check_plane = i845_check_cursor;
14952         } else {
14953                 cursor->max_stride = i9xx_cursor_max_stride;
14954                 cursor->update_plane = i9xx_update_cursor;
14955                 cursor->disable_plane = i9xx_disable_cursor;
14956                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
14957                 cursor->check_plane = i9xx_check_cursor;
14958         }
14959
14960         cursor->cursor.base = ~0;
14961         cursor->cursor.cntl = ~0;
14962
14963         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14964                 cursor->cursor.size = ~0;
14965
14966         possible_crtcs = BIT(pipe);
14967
14968         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14969                                        possible_crtcs, &intel_cursor_plane_funcs,
14970                                        intel_cursor_formats,
14971                                        ARRAY_SIZE(intel_cursor_formats),
14972                                        cursor_format_modifiers,
14973                                        DRM_PLANE_TYPE_CURSOR,
14974                                        "cursor %c", pipe_name(pipe));
14975         if (ret)
14976                 goto fail;
14977
14978         if (INTEL_GEN(dev_priv) >= 4)
14979                 drm_plane_create_rotation_property(&cursor->base,
14980                                                    DRM_MODE_ROTATE_0,
14981                                                    DRM_MODE_ROTATE_0 |
14982                                                    DRM_MODE_ROTATE_180);
14983
14984         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14985
14986         return cursor;
14987
14988 fail:
14989         intel_plane_free(cursor);
14990
14991         return ERR_PTR(ret);
14992 }
14993
14994 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
14995                                     struct intel_crtc_state *crtc_state)
14996 {
14997         struct intel_crtc_scaler_state *scaler_state =
14998                 &crtc_state->scaler_state;
14999         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15000         int i;
15001
15002         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe];
15003         if (!crtc->num_scalers)
15004                 return;
15005
15006         for (i = 0; i < crtc->num_scalers; i++) {
15007                 struct intel_scaler *scaler = &scaler_state->scalers[i];
15008
15009                 scaler->in_use = 0;
15010                 scaler->mode = 0;
15011         }
15012
15013         scaler_state->scaler_id = -1;
15014 }
15015
15016 #define INTEL_CRTC_FUNCS \
15017         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15018         .set_config = drm_atomic_helper_set_config, \
15019         .destroy = intel_crtc_destroy, \
15020         .page_flip = drm_atomic_helper_page_flip, \
15021         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15022         .atomic_destroy_state = intel_crtc_destroy_state, \
15023         .set_crc_source = intel_crtc_set_crc_source, \
15024         .verify_crc_source = intel_crtc_verify_crc_source, \
15025         .get_crc_sources = intel_crtc_get_crc_sources
15026
15027 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15028         INTEL_CRTC_FUNCS,
15029
15030         .get_vblank_counter = g4x_get_vblank_counter,
15031         .enable_vblank = bdw_enable_vblank,
15032         .disable_vblank = bdw_disable_vblank,
15033 };
15034
15035 static const struct drm_crtc_funcs ilk_crtc_funcs = {
15036         INTEL_CRTC_FUNCS,
15037
15038         .get_vblank_counter = g4x_get_vblank_counter,
15039         .enable_vblank = ilk_enable_vblank,
15040         .disable_vblank = ilk_disable_vblank,
15041 };
15042
15043 static const struct drm_crtc_funcs g4x_crtc_funcs = {
15044         INTEL_CRTC_FUNCS,
15045
15046         .get_vblank_counter = g4x_get_vblank_counter,
15047         .enable_vblank = i965_enable_vblank,
15048         .disable_vblank = i965_disable_vblank,
15049 };
15050
15051 static const struct drm_crtc_funcs i965_crtc_funcs = {
15052         INTEL_CRTC_FUNCS,
15053
15054         .get_vblank_counter = i915_get_vblank_counter,
15055         .enable_vblank = i965_enable_vblank,
15056         .disable_vblank = i965_disable_vblank,
15057 };
15058
15059 static const struct drm_crtc_funcs i945gm_crtc_funcs = {
15060         INTEL_CRTC_FUNCS,
15061
15062         .get_vblank_counter = i915_get_vblank_counter,
15063         .enable_vblank = i945gm_enable_vblank,
15064         .disable_vblank = i945gm_disable_vblank,
15065 };
15066
15067 static const struct drm_crtc_funcs i915_crtc_funcs = {
15068         INTEL_CRTC_FUNCS,
15069
15070         .get_vblank_counter = i915_get_vblank_counter,
15071         .enable_vblank = i8xx_enable_vblank,
15072         .disable_vblank = i8xx_disable_vblank,
15073 };
15074
15075 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
15076         INTEL_CRTC_FUNCS,
15077
15078         /* no hw vblank counter */
15079         .enable_vblank = i8xx_enable_vblank,
15080         .disable_vblank = i8xx_disable_vblank,
15081 };
15082
15083 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15084 {
15085         const struct drm_crtc_funcs *funcs;
15086         struct intel_crtc *intel_crtc;
15087         struct intel_crtc_state *crtc_state = NULL;
15088         struct intel_plane *primary = NULL;
15089         struct intel_plane *cursor = NULL;
15090         int sprite, ret;
15091
15092         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
15093         if (!intel_crtc)
15094                 return -ENOMEM;
15095
15096         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
15097         if (!crtc_state) {
15098                 ret = -ENOMEM;
15099                 goto fail;
15100         }
15101         __drm_atomic_helper_crtc_reset(&intel_crtc->base, &crtc_state->base);
15102         intel_crtc->config = crtc_state;
15103
15104         primary = intel_primary_plane_create(dev_priv, pipe);
15105         if (IS_ERR(primary)) {
15106                 ret = PTR_ERR(primary);
15107                 goto fail;
15108         }
15109         intel_crtc->plane_ids_mask |= BIT(primary->id);
15110
15111         for_each_sprite(dev_priv, pipe, sprite) {
15112                 struct intel_plane *plane;
15113
15114                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
15115                 if (IS_ERR(plane)) {
15116                         ret = PTR_ERR(plane);
15117                         goto fail;
15118                 }
15119                 intel_crtc->plane_ids_mask |= BIT(plane->id);
15120         }
15121
15122         cursor = intel_cursor_plane_create(dev_priv, pipe);
15123         if (IS_ERR(cursor)) {
15124                 ret = PTR_ERR(cursor);
15125                 goto fail;
15126         }
15127         intel_crtc->plane_ids_mask |= BIT(cursor->id);
15128
15129         if (HAS_GMCH(dev_priv)) {
15130                 if (IS_CHERRYVIEW(dev_priv) ||
15131                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
15132                         funcs = &g4x_crtc_funcs;
15133                 else if (IS_GEN(dev_priv, 4))
15134                         funcs = &i965_crtc_funcs;
15135                 else if (IS_I945GM(dev_priv))
15136                         funcs = &i945gm_crtc_funcs;
15137                 else if (IS_GEN(dev_priv, 3))
15138                         funcs = &i915_crtc_funcs;
15139                 else
15140                         funcs = &i8xx_crtc_funcs;
15141         } else {
15142                 if (INTEL_GEN(dev_priv) >= 8)
15143                         funcs = &bdw_crtc_funcs;
15144                 else
15145                         funcs = &ilk_crtc_funcs;
15146         }
15147
15148         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
15149                                         &primary->base, &cursor->base,
15150                                         funcs, "pipe %c", pipe_name(pipe));
15151         if (ret)
15152                 goto fail;
15153
15154         intel_crtc->pipe = pipe;
15155
15156         /* initialize shared scalers */
15157         intel_crtc_init_scalers(intel_crtc, crtc_state);
15158
15159         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
15160                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
15161         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
15162
15163         if (INTEL_GEN(dev_priv) < 9) {
15164                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
15165
15166                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
15167                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
15168                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
15169         }
15170
15171         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
15172
15173         intel_color_init(intel_crtc);
15174
15175         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
15176
15177         return 0;
15178
15179 fail:
15180         /*
15181          * drm_mode_config_cleanup() will free up any
15182          * crtcs/planes already initialized.
15183          */
15184         kfree(crtc_state);
15185         kfree(intel_crtc);
15186
15187         return ret;
15188 }
15189
15190 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
15191                                       struct drm_file *file)
15192 {
15193         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
15194         struct drm_crtc *drmmode_crtc;
15195         struct intel_crtc *crtc;
15196
15197         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
15198         if (!drmmode_crtc)
15199                 return -ENOENT;
15200
15201         crtc = to_intel_crtc(drmmode_crtc);
15202         pipe_from_crtc_id->pipe = crtc->pipe;
15203
15204         return 0;
15205 }
15206
15207 static int intel_encoder_clones(struct intel_encoder *encoder)
15208 {
15209         struct drm_device *dev = encoder->base.dev;
15210         struct intel_encoder *source_encoder;
15211         int index_mask = 0;
15212         int entry = 0;
15213
15214         for_each_intel_encoder(dev, source_encoder) {
15215                 if (encoders_cloneable(encoder, source_encoder))
15216                         index_mask |= (1 << entry);
15217
15218                 entry++;
15219         }
15220
15221         return index_mask;
15222 }
15223
15224 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
15225 {
15226         if (!IS_MOBILE(dev_priv))
15227                 return false;
15228
15229         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
15230                 return false;
15231
15232         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
15233                 return false;
15234
15235         return true;
15236 }
15237
15238 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
15239 {
15240         if (INTEL_GEN(dev_priv) >= 9)
15241                 return false;
15242
15243         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
15244                 return false;
15245
15246         if (HAS_PCH_LPT_H(dev_priv) &&
15247             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
15248                 return false;
15249
15250         /* DDI E can't be used if DDI A requires 4 lanes */
15251         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
15252                 return false;
15253
15254         if (!dev_priv->vbt.int_crt_support)
15255                 return false;
15256
15257         return true;
15258 }
15259
15260 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
15261 {
15262         int pps_num;
15263         int pps_idx;
15264
15265         if (HAS_DDI(dev_priv))
15266                 return;
15267         /*
15268          * This w/a is needed at least on CPT/PPT, but to be sure apply it
15269          * everywhere where registers can be write protected.
15270          */
15271         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15272                 pps_num = 2;
15273         else
15274                 pps_num = 1;
15275
15276         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
15277                 u32 val = I915_READ(PP_CONTROL(pps_idx));
15278
15279                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
15280                 I915_WRITE(PP_CONTROL(pps_idx), val);
15281         }
15282 }
15283
15284 static void intel_pps_init(struct drm_i915_private *dev_priv)
15285 {
15286         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
15287                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
15288         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15289                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
15290         else
15291                 dev_priv->pps_mmio_base = PPS_BASE;
15292
15293         intel_pps_unlock_regs_wa(dev_priv);
15294 }
15295
15296 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
15297 {
15298         struct intel_encoder *encoder;
15299         bool dpd_is_edp = false;
15300
15301         intel_pps_init(dev_priv);
15302
15303         if (!HAS_DISPLAY(dev_priv))
15304                 return;
15305
15306         if (INTEL_GEN(dev_priv) >= 12) {
15307                 /* TODO: initialize TC ports as well */
15308                 intel_ddi_init(dev_priv, PORT_A);
15309                 intel_ddi_init(dev_priv, PORT_B);
15310                 icl_dsi_init(dev_priv);
15311         } else if (IS_ELKHARTLAKE(dev_priv)) {
15312                 intel_ddi_init(dev_priv, PORT_A);
15313                 intel_ddi_init(dev_priv, PORT_B);
15314                 intel_ddi_init(dev_priv, PORT_C);
15315                 intel_ddi_init(dev_priv, PORT_D);
15316                 icl_dsi_init(dev_priv);
15317         } else if (IS_GEN(dev_priv, 11)) {
15318                 intel_ddi_init(dev_priv, PORT_A);
15319                 intel_ddi_init(dev_priv, PORT_B);
15320                 intel_ddi_init(dev_priv, PORT_C);
15321                 intel_ddi_init(dev_priv, PORT_D);
15322                 intel_ddi_init(dev_priv, PORT_E);
15323                 /*
15324                  * On some ICL SKUs port F is not present. No strap bits for
15325                  * this, so rely on VBT.
15326                  * Work around broken VBTs on SKUs known to have no port F.
15327                  */
15328                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
15329                     intel_bios_is_port_present(dev_priv, PORT_F))
15330                         intel_ddi_init(dev_priv, PORT_F);
15331
15332                 icl_dsi_init(dev_priv);
15333         } else if (IS_GEN9_LP(dev_priv)) {
15334                 /*
15335                  * FIXME: Broxton doesn't support port detection via the
15336                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
15337                  * detect the ports.
15338                  */
15339                 intel_ddi_init(dev_priv, PORT_A);
15340                 intel_ddi_init(dev_priv, PORT_B);
15341                 intel_ddi_init(dev_priv, PORT_C);
15342
15343                 vlv_dsi_init(dev_priv);
15344         } else if (HAS_DDI(dev_priv)) {
15345                 int found;
15346
15347                 if (intel_ddi_crt_present(dev_priv))
15348                         intel_crt_init(dev_priv);
15349
15350                 /*
15351                  * Haswell uses DDI functions to detect digital outputs.
15352                  * On SKL pre-D0 the strap isn't connected, so we assume
15353                  * it's there.
15354                  */
15355                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
15356                 /* WaIgnoreDDIAStrap: skl */
15357                 if (found || IS_GEN9_BC(dev_priv))
15358                         intel_ddi_init(dev_priv, PORT_A);
15359
15360                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
15361                  * register */
15362                 found = I915_READ(SFUSE_STRAP);
15363
15364                 if (found & SFUSE_STRAP_DDIB_DETECTED)
15365                         intel_ddi_init(dev_priv, PORT_B);
15366                 if (found & SFUSE_STRAP_DDIC_DETECTED)
15367                         intel_ddi_init(dev_priv, PORT_C);
15368                 if (found & SFUSE_STRAP_DDID_DETECTED)
15369                         intel_ddi_init(dev_priv, PORT_D);
15370                 if (found & SFUSE_STRAP_DDIF_DETECTED)
15371                         intel_ddi_init(dev_priv, PORT_F);
15372                 /*
15373                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
15374                  */
15375                 if (IS_GEN9_BC(dev_priv) &&
15376                     intel_bios_is_port_present(dev_priv, PORT_E))
15377                         intel_ddi_init(dev_priv, PORT_E);
15378
15379         } else if (HAS_PCH_SPLIT(dev_priv)) {
15380                 int found;
15381
15382                 /*
15383                  * intel_edp_init_connector() depends on this completing first,
15384                  * to prevent the registration of both eDP and LVDS and the
15385                  * incorrect sharing of the PPS.
15386                  */
15387                 intel_lvds_init(dev_priv);
15388                 intel_crt_init(dev_priv);
15389
15390                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
15391
15392                 if (ilk_has_edp_a(dev_priv))
15393                         intel_dp_init(dev_priv, DP_A, PORT_A);
15394
15395                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
15396                         /* PCH SDVOB multiplex with HDMIB */
15397                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
15398                         if (!found)
15399                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
15400                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
15401                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
15402                 }
15403
15404                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
15405                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
15406
15407                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
15408                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
15409
15410                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
15411                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
15412
15413                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
15414                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
15415         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15416                 bool has_edp, has_port;
15417
15418                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
15419                         intel_crt_init(dev_priv);
15420
15421                 /*
15422                  * The DP_DETECTED bit is the latched state of the DDC
15423                  * SDA pin at boot. However since eDP doesn't require DDC
15424                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
15425                  * eDP ports may have been muxed to an alternate function.
15426                  * Thus we can't rely on the DP_DETECTED bit alone to detect
15427                  * eDP ports. Consult the VBT as well as DP_DETECTED to
15428                  * detect eDP ports.
15429                  *
15430                  * Sadly the straps seem to be missing sometimes even for HDMI
15431                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
15432                  * and VBT for the presence of the port. Additionally we can't
15433                  * trust the port type the VBT declares as we've seen at least
15434                  * HDMI ports that the VBT claim are DP or eDP.
15435                  */
15436                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
15437                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
15438                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
15439                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
15440                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
15441                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
15442
15443                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
15444                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
15445                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
15446                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
15447                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
15448                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
15449
15450                 if (IS_CHERRYVIEW(dev_priv)) {
15451                         /*
15452                          * eDP not supported on port D,
15453                          * so no need to worry about it
15454                          */
15455                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
15456                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
15457                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
15458                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
15459                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
15460                 }
15461
15462                 vlv_dsi_init(dev_priv);
15463         } else if (IS_PINEVIEW(dev_priv)) {
15464                 intel_lvds_init(dev_priv);
15465                 intel_crt_init(dev_priv);
15466         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
15467                 bool found = false;
15468
15469                 if (IS_MOBILE(dev_priv))
15470                         intel_lvds_init(dev_priv);
15471
15472                 intel_crt_init(dev_priv);
15473
15474                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15475                         DRM_DEBUG_KMS("probing SDVOB\n");
15476                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
15477                         if (!found && IS_G4X(dev_priv)) {
15478                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
15479                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
15480                         }
15481
15482                         if (!found && IS_G4X(dev_priv))
15483                                 intel_dp_init(dev_priv, DP_B, PORT_B);
15484                 }
15485
15486                 /* Before G4X SDVOC doesn't have its own detect register */
15487
15488                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15489                         DRM_DEBUG_KMS("probing SDVOC\n");
15490                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
15491                 }
15492
15493                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
15494
15495                         if (IS_G4X(dev_priv)) {
15496                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
15497                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
15498                         }
15499                         if (IS_G4X(dev_priv))
15500                                 intel_dp_init(dev_priv, DP_C, PORT_C);
15501                 }
15502
15503                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
15504                         intel_dp_init(dev_priv, DP_D, PORT_D);
15505
15506                 if (SUPPORTS_TV(dev_priv))
15507                         intel_tv_init(dev_priv);
15508         } else if (IS_GEN(dev_priv, 2)) {
15509                 if (IS_I85X(dev_priv))
15510                         intel_lvds_init(dev_priv);
15511
15512                 intel_crt_init(dev_priv);
15513                 intel_dvo_init(dev_priv);
15514         }
15515
15516         intel_psr_init(dev_priv);
15517
15518         for_each_intel_encoder(&dev_priv->drm, encoder) {
15519                 encoder->base.possible_crtcs = encoder->crtc_mask;
15520                 encoder->base.possible_clones =
15521                         intel_encoder_clones(encoder);
15522         }
15523
15524         intel_init_pch_refclk(dev_priv);
15525
15526         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
15527 }
15528
15529 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
15530 {
15531         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
15532
15533         drm_framebuffer_cleanup(fb);
15534         intel_frontbuffer_put(intel_fb->frontbuffer);
15535
15536         kfree(intel_fb);
15537 }
15538
15539 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
15540                                                 struct drm_file *file,
15541                                                 unsigned int *handle)
15542 {
15543         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15544
15545         if (obj->userptr.mm) {
15546                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
15547                 return -EINVAL;
15548         }
15549
15550         return drm_gem_handle_create(file, &obj->base, handle);
15551 }
15552
15553 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
15554                                         struct drm_file *file,
15555                                         unsigned flags, unsigned color,
15556                                         struct drm_clip_rect *clips,
15557                                         unsigned num_clips)
15558 {
15559         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15560
15561         i915_gem_object_flush_if_display(obj);
15562         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
15563
15564         return 0;
15565 }
15566
15567 static const struct drm_framebuffer_funcs intel_fb_funcs = {
15568         .destroy = intel_user_framebuffer_destroy,
15569         .create_handle = intel_user_framebuffer_create_handle,
15570         .dirty = intel_user_framebuffer_dirty,
15571 };
15572
15573 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
15574                                   struct drm_i915_gem_object *obj,
15575                                   struct drm_mode_fb_cmd2 *mode_cmd)
15576 {
15577         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
15578         struct drm_framebuffer *fb = &intel_fb->base;
15579         u32 max_stride;
15580         unsigned int tiling, stride;
15581         int ret = -EINVAL;
15582         int i;
15583
15584         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
15585         if (!intel_fb->frontbuffer)
15586                 return -ENOMEM;
15587
15588         i915_gem_object_lock(obj);
15589         tiling = i915_gem_object_get_tiling(obj);
15590         stride = i915_gem_object_get_stride(obj);
15591         i915_gem_object_unlock(obj);
15592
15593         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
15594                 /*
15595                  * If there's a fence, enforce that
15596                  * the fb modifier and tiling mode match.
15597                  */
15598                 if (tiling != I915_TILING_NONE &&
15599                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15600                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
15601                         goto err;
15602                 }
15603         } else {
15604                 if (tiling == I915_TILING_X) {
15605                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
15606                 } else if (tiling == I915_TILING_Y) {
15607                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
15608                         goto err;
15609                 }
15610         }
15611
15612         if (!drm_any_plane_has_format(&dev_priv->drm,
15613                                       mode_cmd->pixel_format,
15614                                       mode_cmd->modifier[0])) {
15615                 struct drm_format_name_buf format_name;
15616
15617                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
15618                               drm_get_format_name(mode_cmd->pixel_format,
15619                                                   &format_name),
15620                               mode_cmd->modifier[0]);
15621                 goto err;
15622         }
15623
15624         /*
15625          * gen2/3 display engine uses the fence if present,
15626          * so the tiling mode must match the fb modifier exactly.
15627          */
15628         if (INTEL_GEN(dev_priv) < 4 &&
15629             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15630                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
15631                 goto err;
15632         }
15633
15634         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
15635                                          mode_cmd->modifier[0]);
15636         if (mode_cmd->pitches[0] > max_stride) {
15637                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
15638                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
15639                               "tiled" : "linear",
15640                               mode_cmd->pitches[0], max_stride);
15641                 goto err;
15642         }
15643
15644         /*
15645          * If there's a fence, enforce that
15646          * the fb pitch and fence stride match.
15647          */
15648         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
15649                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
15650                               mode_cmd->pitches[0], stride);
15651                 goto err;
15652         }
15653
15654         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
15655         if (mode_cmd->offsets[0] != 0)
15656                 goto err;
15657
15658         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
15659
15660         for (i = 0; i < fb->format->num_planes; i++) {
15661                 u32 stride_alignment;
15662
15663                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
15664                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
15665                         goto err;
15666                 }
15667
15668                 stride_alignment = intel_fb_stride_alignment(fb, i);
15669
15670                 /*
15671                  * Display WA #0531: skl,bxt,kbl,glk
15672                  *
15673                  * Render decompression and plane width > 3840
15674                  * combined with horizontal panning requires the
15675                  * plane stride to be a multiple of 4. We'll just
15676                  * require the entire fb to accommodate that to avoid
15677                  * potential runtime errors at plane configuration time.
15678                  */
15679                 if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 &&
15680                     is_ccs_modifier(fb->modifier))
15681                         stride_alignment *= 4;
15682
15683                 if (fb->pitches[i] & (stride_alignment - 1)) {
15684                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
15685                                       i, fb->pitches[i], stride_alignment);
15686                         goto err;
15687                 }
15688
15689                 fb->obj[i] = &obj->base;
15690         }
15691
15692         ret = intel_fill_fb_info(dev_priv, fb);
15693         if (ret)
15694                 goto err;
15695
15696         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
15697         if (ret) {
15698                 DRM_ERROR("framebuffer init failed %d\n", ret);
15699                 goto err;
15700         }
15701
15702         return 0;
15703
15704 err:
15705         intel_frontbuffer_put(intel_fb->frontbuffer);
15706         return ret;
15707 }
15708
15709 static struct drm_framebuffer *
15710 intel_user_framebuffer_create(struct drm_device *dev,
15711                               struct drm_file *filp,
15712                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
15713 {
15714         struct drm_framebuffer *fb;
15715         struct drm_i915_gem_object *obj;
15716         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
15717
15718         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
15719         if (!obj)
15720                 return ERR_PTR(-ENOENT);
15721
15722         fb = intel_framebuffer_create(obj, &mode_cmd);
15723         i915_gem_object_put(obj);
15724
15725         return fb;
15726 }
15727
15728 static void intel_atomic_state_free(struct drm_atomic_state *state)
15729 {
15730         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
15731
15732         drm_atomic_state_default_release(state);
15733
15734         i915_sw_fence_fini(&intel_state->commit_ready);
15735
15736         kfree(state);
15737 }
15738
15739 static enum drm_mode_status
15740 intel_mode_valid(struct drm_device *dev,
15741                  const struct drm_display_mode *mode)
15742 {
15743         struct drm_i915_private *dev_priv = to_i915(dev);
15744         int hdisplay_max, htotal_max;
15745         int vdisplay_max, vtotal_max;
15746
15747         /*
15748          * Can't reject DBLSCAN here because Xorg ddxen can add piles
15749          * of DBLSCAN modes to the output's mode list when they detect
15750          * the scaling mode property on the connector. And they don't
15751          * ask the kernel to validate those modes in any way until
15752          * modeset time at which point the client gets a protocol error.
15753          * So in order to not upset those clients we silently ignore the
15754          * DBLSCAN flag on such connectors. For other connectors we will
15755          * reject modes with the DBLSCAN flag in encoder->compute_config().
15756          * And we always reject DBLSCAN modes in connector->mode_valid()
15757          * as we never want such modes on the connector's mode list.
15758          */
15759
15760         if (mode->vscan > 1)
15761                 return MODE_NO_VSCAN;
15762
15763         if (mode->flags & DRM_MODE_FLAG_HSKEW)
15764                 return MODE_H_ILLEGAL;
15765
15766         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
15767                            DRM_MODE_FLAG_NCSYNC |
15768                            DRM_MODE_FLAG_PCSYNC))
15769                 return MODE_HSYNC;
15770
15771         if (mode->flags & (DRM_MODE_FLAG_BCAST |
15772                            DRM_MODE_FLAG_PIXMUX |
15773                            DRM_MODE_FLAG_CLKDIV2))
15774                 return MODE_BAD;
15775
15776         if (INTEL_GEN(dev_priv) >= 9 ||
15777             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
15778                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
15779                 vdisplay_max = 4096;
15780                 htotal_max = 8192;
15781                 vtotal_max = 8192;
15782         } else if (INTEL_GEN(dev_priv) >= 3) {
15783                 hdisplay_max = 4096;
15784                 vdisplay_max = 4096;
15785                 htotal_max = 8192;
15786                 vtotal_max = 8192;
15787         } else {
15788                 hdisplay_max = 2048;
15789                 vdisplay_max = 2048;
15790                 htotal_max = 4096;
15791                 vtotal_max = 4096;
15792         }
15793
15794         if (mode->hdisplay > hdisplay_max ||
15795             mode->hsync_start > htotal_max ||
15796             mode->hsync_end > htotal_max ||
15797             mode->htotal > htotal_max)
15798                 return MODE_H_ILLEGAL;
15799
15800         if (mode->vdisplay > vdisplay_max ||
15801             mode->vsync_start > vtotal_max ||
15802             mode->vsync_end > vtotal_max ||
15803             mode->vtotal > vtotal_max)
15804                 return MODE_V_ILLEGAL;
15805
15806         return MODE_OK;
15807 }
15808
15809 static const struct drm_mode_config_funcs intel_mode_funcs = {
15810         .fb_create = intel_user_framebuffer_create,
15811         .get_format_info = intel_get_format_info,
15812         .output_poll_changed = intel_fbdev_output_poll_changed,
15813         .mode_valid = intel_mode_valid,
15814         .atomic_check = intel_atomic_check,
15815         .atomic_commit = intel_atomic_commit,
15816         .atomic_state_alloc = intel_atomic_state_alloc,
15817         .atomic_state_clear = intel_atomic_state_clear,
15818         .atomic_state_free = intel_atomic_state_free,
15819 };
15820
15821 /**
15822  * intel_init_display_hooks - initialize the display modesetting hooks
15823  * @dev_priv: device private
15824  */
15825 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
15826 {
15827         intel_init_cdclk_hooks(dev_priv);
15828
15829         if (INTEL_GEN(dev_priv) >= 9) {
15830                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15831                 dev_priv->display.get_initial_plane_config =
15832                         skylake_get_initial_plane_config;
15833                 dev_priv->display.crtc_compute_clock =
15834                         haswell_crtc_compute_clock;
15835                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15836                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15837         } else if (HAS_DDI(dev_priv)) {
15838                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15839                 dev_priv->display.get_initial_plane_config =
15840                         i9xx_get_initial_plane_config;
15841                 dev_priv->display.crtc_compute_clock =
15842                         haswell_crtc_compute_clock;
15843                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15844                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15845         } else if (HAS_PCH_SPLIT(dev_priv)) {
15846                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
15847                 dev_priv->display.get_initial_plane_config =
15848                         i9xx_get_initial_plane_config;
15849                 dev_priv->display.crtc_compute_clock =
15850                         ironlake_crtc_compute_clock;
15851                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
15852                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
15853         } else if (IS_CHERRYVIEW(dev_priv)) {
15854                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15855                 dev_priv->display.get_initial_plane_config =
15856                         i9xx_get_initial_plane_config;
15857                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
15858                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15859                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15860         } else if (IS_VALLEYVIEW(dev_priv)) {
15861                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15862                 dev_priv->display.get_initial_plane_config =
15863                         i9xx_get_initial_plane_config;
15864                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
15865                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15866                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15867         } else if (IS_G4X(dev_priv)) {
15868                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15869                 dev_priv->display.get_initial_plane_config =
15870                         i9xx_get_initial_plane_config;
15871                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
15872                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15873                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15874         } else if (IS_PINEVIEW(dev_priv)) {
15875                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15876                 dev_priv->display.get_initial_plane_config =
15877                         i9xx_get_initial_plane_config;
15878                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
15879                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15880                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15881         } else if (!IS_GEN(dev_priv, 2)) {
15882                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15883                 dev_priv->display.get_initial_plane_config =
15884                         i9xx_get_initial_plane_config;
15885                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
15886                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15887                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15888         } else {
15889                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15890                 dev_priv->display.get_initial_plane_config =
15891                         i9xx_get_initial_plane_config;
15892                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
15893                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15894                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15895         }
15896
15897         if (IS_GEN(dev_priv, 5)) {
15898                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
15899         } else if (IS_GEN(dev_priv, 6)) {
15900                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
15901         } else if (IS_IVYBRIDGE(dev_priv)) {
15902                 /* FIXME: detect B0+ stepping and use auto training */
15903                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
15904         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
15905                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
15906         }
15907
15908         if (INTEL_GEN(dev_priv) >= 9)
15909                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
15910         else
15911                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
15912 }
15913
15914 static i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
15915 {
15916         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15917                 return VLV_VGACNTRL;
15918         else if (INTEL_GEN(dev_priv) >= 5)
15919                 return CPU_VGACNTRL;
15920         else
15921                 return VGACNTRL;
15922 }
15923
15924 /* Disable the VGA plane that we never use */
15925 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15926 {
15927         struct pci_dev *pdev = dev_priv->drm.pdev;
15928         u8 sr1;
15929         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15930
15931         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15932         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15933         outb(SR01, VGA_SR_INDEX);
15934         sr1 = inb(VGA_SR_DATA);
15935         outb(sr1 | 1<<5, VGA_SR_DATA);
15936         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15937         udelay(300);
15938
15939         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15940         POSTING_READ(vga_reg);
15941 }
15942
15943 void intel_modeset_init_hw(struct drm_device *dev)
15944 {
15945         struct drm_i915_private *dev_priv = to_i915(dev);
15946
15947         intel_update_cdclk(dev_priv);
15948         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15949         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15950 }
15951
15952 /*
15953  * Calculate what we think the watermarks should be for the state we've read
15954  * out of the hardware and then immediately program those watermarks so that
15955  * we ensure the hardware settings match our internal state.
15956  *
15957  * We can calculate what we think WM's should be by creating a duplicate of the
15958  * current state (which was constructed during hardware readout) and running it
15959  * through the atomic check code to calculate new watermark values in the
15960  * state object.
15961  */
15962 static void sanitize_watermarks(struct drm_device *dev)
15963 {
15964         struct drm_i915_private *dev_priv = to_i915(dev);
15965         struct drm_atomic_state *state;
15966         struct intel_atomic_state *intel_state;
15967         struct intel_crtc *crtc;
15968         struct intel_crtc_state *crtc_state;
15969         struct drm_modeset_acquire_ctx ctx;
15970         int ret;
15971         int i;
15972
15973         /* Only supported on platforms that use atomic watermark design */
15974         if (!dev_priv->display.optimize_watermarks)
15975                 return;
15976
15977         /*
15978          * We need to hold connection_mutex before calling duplicate_state so
15979          * that the connector loop is protected.
15980          */
15981         drm_modeset_acquire_init(&ctx, 0);
15982 retry:
15983         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15984         if (ret == -EDEADLK) {
15985                 drm_modeset_backoff(&ctx);
15986                 goto retry;
15987         } else if (WARN_ON(ret)) {
15988                 goto fail;
15989         }
15990
15991         state = drm_atomic_helper_duplicate_state(dev, &ctx);
15992         if (WARN_ON(IS_ERR(state)))
15993                 goto fail;
15994
15995         intel_state = to_intel_atomic_state(state);
15996
15997         /*
15998          * Hardware readout is the only time we don't want to calculate
15999          * intermediate watermarks (since we don't trust the current
16000          * watermarks).
16001          */
16002         if (!HAS_GMCH(dev_priv))
16003                 intel_state->skip_intermediate_wm = true;
16004
16005         ret = intel_atomic_check(dev, state);
16006         if (ret) {
16007                 /*
16008                  * If we fail here, it means that the hardware appears to be
16009                  * programmed in a way that shouldn't be possible, given our
16010                  * understanding of watermark requirements.  This might mean a
16011                  * mistake in the hardware readout code or a mistake in the
16012                  * watermark calculations for a given platform.  Raise a WARN
16013                  * so that this is noticeable.
16014                  *
16015                  * If this actually happens, we'll have to just leave the
16016                  * BIOS-programmed watermarks untouched and hope for the best.
16017                  */
16018                 WARN(true, "Could not determine valid watermarks for inherited state\n");
16019                 goto put_state;
16020         }
16021
16022         /* Write calculated watermark values back */
16023         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
16024                 crtc_state->wm.need_postvbl_update = true;
16025                 dev_priv->display.optimize_watermarks(intel_state, crtc_state);
16026
16027                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
16028         }
16029
16030 put_state:
16031         drm_atomic_state_put(state);
16032 fail:
16033         drm_modeset_drop_locks(&ctx);
16034         drm_modeset_acquire_fini(&ctx);
16035 }
16036
16037 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
16038 {
16039         if (IS_GEN(dev_priv, 5)) {
16040                 u32 fdi_pll_clk =
16041                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
16042
16043                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
16044         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
16045                 dev_priv->fdi_pll_freq = 270000;
16046         } else {
16047                 return;
16048         }
16049
16050         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
16051 }
16052
16053 static int intel_initial_commit(struct drm_device *dev)
16054 {
16055         struct drm_atomic_state *state = NULL;
16056         struct drm_modeset_acquire_ctx ctx;
16057         struct drm_crtc *crtc;
16058         struct drm_crtc_state *crtc_state;
16059         int ret = 0;
16060
16061         state = drm_atomic_state_alloc(dev);
16062         if (!state)
16063                 return -ENOMEM;
16064
16065         drm_modeset_acquire_init(&ctx, 0);
16066
16067 retry:
16068         state->acquire_ctx = &ctx;
16069
16070         drm_for_each_crtc(crtc, dev) {
16071                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
16072                 if (IS_ERR(crtc_state)) {
16073                         ret = PTR_ERR(crtc_state);
16074                         goto out;
16075                 }
16076
16077                 if (crtc_state->active) {
16078                         ret = drm_atomic_add_affected_planes(state, crtc);
16079                         if (ret)
16080                                 goto out;
16081
16082                         /*
16083                          * FIXME hack to force a LUT update to avoid the
16084                          * plane update forcing the pipe gamma on without
16085                          * having a proper LUT loaded. Remove once we
16086                          * have readout for pipe gamma enable.
16087                          */
16088                         crtc_state->color_mgmt_changed = true;
16089                 }
16090         }
16091
16092         ret = drm_atomic_commit(state);
16093
16094 out:
16095         if (ret == -EDEADLK) {
16096                 drm_atomic_state_clear(state);
16097                 drm_modeset_backoff(&ctx);
16098                 goto retry;
16099         }
16100
16101         drm_atomic_state_put(state);
16102
16103         drm_modeset_drop_locks(&ctx);
16104         drm_modeset_acquire_fini(&ctx);
16105
16106         return ret;
16107 }
16108
16109 int intel_modeset_init(struct drm_device *dev)
16110 {
16111         struct drm_i915_private *dev_priv = to_i915(dev);
16112         enum pipe pipe;
16113         struct intel_crtc *crtc;
16114         int ret;
16115
16116         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
16117
16118         drm_mode_config_init(dev);
16119
16120         ret = intel_bw_init(dev_priv);
16121         if (ret)
16122                 return ret;
16123
16124         dev->mode_config.min_width = 0;
16125         dev->mode_config.min_height = 0;
16126
16127         dev->mode_config.preferred_depth = 24;
16128         dev->mode_config.prefer_shadow = 1;
16129
16130         dev->mode_config.allow_fb_modifiers = true;
16131
16132         dev->mode_config.funcs = &intel_mode_funcs;
16133
16134         init_llist_head(&dev_priv->atomic_helper.free_list);
16135         INIT_WORK(&dev_priv->atomic_helper.free_work,
16136                   intel_atomic_helper_free_state_worker);
16137
16138         intel_init_quirks(dev_priv);
16139
16140         intel_fbc_init(dev_priv);
16141
16142         intel_init_pm(dev_priv);
16143
16144         /*
16145          * There may be no VBT; and if the BIOS enabled SSC we can
16146          * just keep using it to avoid unnecessary flicker.  Whereas if the
16147          * BIOS isn't using it, don't assume it will work even if the VBT
16148          * indicates as much.
16149          */
16150         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
16151                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
16152                                             DREF_SSC1_ENABLE);
16153
16154                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
16155                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
16156                                      bios_lvds_use_ssc ? "en" : "dis",
16157                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
16158                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
16159                 }
16160         }
16161
16162         /*
16163          * Maximum framebuffer dimensions, chosen to match
16164          * the maximum render engine surface size on gen4+.
16165          */
16166         if (INTEL_GEN(dev_priv) >= 7) {
16167                 dev->mode_config.max_width = 16384;
16168                 dev->mode_config.max_height = 16384;
16169         } else if (INTEL_GEN(dev_priv) >= 4) {
16170                 dev->mode_config.max_width = 8192;
16171                 dev->mode_config.max_height = 8192;
16172         } else if (IS_GEN(dev_priv, 3)) {
16173                 dev->mode_config.max_width = 4096;
16174                 dev->mode_config.max_height = 4096;
16175         } else {
16176                 dev->mode_config.max_width = 2048;
16177                 dev->mode_config.max_height = 2048;
16178         }
16179
16180         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16181                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
16182                 dev->mode_config.cursor_height = 1023;
16183         } else if (IS_GEN(dev_priv, 2)) {
16184                 dev->mode_config.cursor_width = 64;
16185                 dev->mode_config.cursor_height = 64;
16186         } else {
16187                 dev->mode_config.cursor_width = 256;
16188                 dev->mode_config.cursor_height = 256;
16189         }
16190
16191         DRM_DEBUG_KMS("%d display pipe%s available.\n",
16192                       INTEL_INFO(dev_priv)->num_pipes,
16193                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
16194
16195         for_each_pipe(dev_priv, pipe) {
16196                 ret = intel_crtc_init(dev_priv, pipe);
16197                 if (ret) {
16198                         drm_mode_config_cleanup(dev);
16199                         return ret;
16200                 }
16201         }
16202
16203         intel_shared_dpll_init(dev);
16204         intel_update_fdi_pll_freq(dev_priv);
16205
16206         intel_update_czclk(dev_priv);
16207         intel_modeset_init_hw(dev);
16208
16209         intel_hdcp_component_init(dev_priv);
16210
16211         if (dev_priv->max_cdclk_freq == 0)
16212                 intel_update_max_cdclk(dev_priv);
16213
16214         /* Just disable it once at startup */
16215         i915_disable_vga(dev_priv);
16216         intel_setup_outputs(dev_priv);
16217
16218         drm_modeset_lock_all(dev);
16219         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
16220         drm_modeset_unlock_all(dev);
16221
16222         for_each_intel_crtc(dev, crtc) {
16223                 struct intel_initial_plane_config plane_config = {};
16224
16225                 if (!crtc->active)
16226                         continue;
16227
16228                 /*
16229                  * Note that reserving the BIOS fb up front prevents us
16230                  * from stuffing other stolen allocations like the ring
16231                  * on top.  This prevents some ugliness at boot time, and
16232                  * can even allow for smooth boot transitions if the BIOS
16233                  * fb is large enough for the active pipe configuration.
16234                  */
16235                 dev_priv->display.get_initial_plane_config(crtc,
16236                                                            &plane_config);
16237
16238                 /*
16239                  * If the fb is shared between multiple heads, we'll
16240                  * just get the first one.
16241                  */
16242                 intel_find_initial_plane_obj(crtc, &plane_config);
16243         }
16244
16245         /*
16246          * Make sure hardware watermarks really match the state we read out.
16247          * Note that we need to do this after reconstructing the BIOS fb's
16248          * since the watermark calculation done here will use pstate->fb.
16249          */
16250         if (!HAS_GMCH(dev_priv))
16251                 sanitize_watermarks(dev);
16252
16253         /*
16254          * Force all active planes to recompute their states. So that on
16255          * mode_setcrtc after probe, all the intel_plane_state variables
16256          * are already calculated and there is no assert_plane warnings
16257          * during bootup.
16258          */
16259         ret = intel_initial_commit(dev);
16260         if (ret)
16261                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
16262
16263         return 0;
16264 }
16265
16266 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16267 {
16268         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16269         /* 640x480@60Hz, ~25175 kHz */
16270         struct dpll clock = {
16271                 .m1 = 18,
16272                 .m2 = 7,
16273                 .p1 = 13,
16274                 .p2 = 4,
16275                 .n = 2,
16276         };
16277         u32 dpll, fp;
16278         int i;
16279
16280         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
16281
16282         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
16283                       pipe_name(pipe), clock.vco, clock.dot);
16284
16285         fp = i9xx_dpll_compute_fp(&clock);
16286         dpll = DPLL_DVO_2X_MODE |
16287                 DPLL_VGA_MODE_DIS |
16288                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
16289                 PLL_P2_DIVIDE_BY_4 |
16290                 PLL_REF_INPUT_DREFCLK |
16291                 DPLL_VCO_ENABLE;
16292
16293         I915_WRITE(FP0(pipe), fp);
16294         I915_WRITE(FP1(pipe), fp);
16295
16296         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
16297         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
16298         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
16299         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
16300         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
16301         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
16302         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
16303
16304         /*
16305          * Apparently we need to have VGA mode enabled prior to changing
16306          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
16307          * dividers, even though the register value does change.
16308          */
16309         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
16310         I915_WRITE(DPLL(pipe), dpll);
16311
16312         /* Wait for the clocks to stabilize. */
16313         POSTING_READ(DPLL(pipe));
16314         udelay(150);
16315
16316         /* The pixel multiplier can only be updated once the
16317          * DPLL is enabled and the clocks are stable.
16318          *
16319          * So write it again.
16320          */
16321         I915_WRITE(DPLL(pipe), dpll);
16322
16323         /* We do this three times for luck */
16324         for (i = 0; i < 3 ; i++) {
16325                 I915_WRITE(DPLL(pipe), dpll);
16326                 POSTING_READ(DPLL(pipe));
16327                 udelay(150); /* wait for warmup */
16328         }
16329
16330         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
16331         POSTING_READ(PIPECONF(pipe));
16332
16333         intel_wait_for_pipe_scanline_moving(crtc);
16334 }
16335
16336 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16337 {
16338         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16339
16340         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
16341                       pipe_name(pipe));
16342
16343         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
16344         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
16345         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
16346         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
16347         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
16348
16349         I915_WRITE(PIPECONF(pipe), 0);
16350         POSTING_READ(PIPECONF(pipe));
16351
16352         intel_wait_for_pipe_scanline_stopped(crtc);
16353
16354         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
16355         POSTING_READ(DPLL(pipe));
16356 }
16357
16358 static void
16359 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
16360 {
16361         struct intel_crtc *crtc;
16362
16363         if (INTEL_GEN(dev_priv) >= 4)
16364                 return;
16365
16366         for_each_intel_crtc(&dev_priv->drm, crtc) {
16367                 struct intel_plane *plane =
16368                         to_intel_plane(crtc->base.primary);
16369                 struct intel_crtc *plane_crtc;
16370                 enum pipe pipe;
16371
16372                 if (!plane->get_hw_state(plane, &pipe))
16373                         continue;
16374
16375                 if (pipe == crtc->pipe)
16376                         continue;
16377
16378                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
16379                               plane->base.base.id, plane->base.name);
16380
16381                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16382                 intel_plane_disable_noatomic(plane_crtc, plane);
16383         }
16384 }
16385
16386 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
16387 {
16388         struct drm_device *dev = crtc->base.dev;
16389         struct intel_encoder *encoder;
16390
16391         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
16392                 return true;
16393
16394         return false;
16395 }
16396
16397 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
16398 {
16399         struct drm_device *dev = encoder->base.dev;
16400         struct intel_connector *connector;
16401
16402         for_each_connector_on_encoder(dev, &encoder->base, connector)
16403                 return connector;
16404
16405         return NULL;
16406 }
16407
16408 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
16409                               enum pipe pch_transcoder)
16410 {
16411         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
16412                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
16413 }
16414
16415 static void intel_sanitize_crtc(struct intel_crtc *crtc,
16416                                 struct drm_modeset_acquire_ctx *ctx)
16417 {
16418         struct drm_device *dev = crtc->base.dev;
16419         struct drm_i915_private *dev_priv = to_i915(dev);
16420         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
16421         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
16422
16423         /* Clear any frame start delays used for debugging left by the BIOS */
16424         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
16425                 i915_reg_t reg = PIPECONF(cpu_transcoder);
16426
16427                 I915_WRITE(reg,
16428                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
16429         }
16430
16431         if (crtc_state->base.active) {
16432                 struct intel_plane *plane;
16433
16434                 /* Disable everything but the primary plane */
16435                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
16436                         const struct intel_plane_state *plane_state =
16437                                 to_intel_plane_state(plane->base.state);
16438
16439                         if (plane_state->base.visible &&
16440                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
16441                                 intel_plane_disable_noatomic(crtc, plane);
16442                 }
16443
16444                 /*
16445                  * Disable any background color set by the BIOS, but enable the
16446                  * gamma and CSC to match how we program our planes.
16447                  */
16448                 if (INTEL_GEN(dev_priv) >= 9)
16449                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
16450                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
16451                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
16452         }
16453
16454         /* Adjust the state of the output pipe according to whether we
16455          * have active connectors/encoders. */
16456         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
16457                 intel_crtc_disable_noatomic(&crtc->base, ctx);
16458
16459         if (crtc_state->base.active || HAS_GMCH(dev_priv)) {
16460                 /*
16461                  * We start out with underrun reporting disabled to avoid races.
16462                  * For correct bookkeeping mark this on active crtcs.
16463                  *
16464                  * Also on gmch platforms we dont have any hardware bits to
16465                  * disable the underrun reporting. Which means we need to start
16466                  * out with underrun reporting disabled also on inactive pipes,
16467                  * since otherwise we'll complain about the garbage we read when
16468                  * e.g. coming up after runtime pm.
16469                  *
16470                  * No protection against concurrent access is required - at
16471                  * worst a fifo underrun happens which also sets this to false.
16472                  */
16473                 crtc->cpu_fifo_underrun_disabled = true;
16474                 /*
16475                  * We track the PCH trancoder underrun reporting state
16476                  * within the crtc. With crtc for pipe A housing the underrun
16477                  * reporting state for PCH transcoder A, crtc for pipe B housing
16478                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
16479                  * and marking underrun reporting as disabled for the non-existing
16480                  * PCH transcoders B and C would prevent enabling the south
16481                  * error interrupt (see cpt_can_enable_serr_int()).
16482                  */
16483                 if (has_pch_trancoder(dev_priv, crtc->pipe))
16484                         crtc->pch_fifo_underrun_disabled = true;
16485         }
16486 }
16487
16488 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
16489 {
16490         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
16491
16492         /*
16493          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
16494          * the hardware when a high res displays plugged in. DPLL P
16495          * divider is zero, and the pipe timings are bonkers. We'll
16496          * try to disable everything in that case.
16497          *
16498          * FIXME would be nice to be able to sanitize this state
16499          * without several WARNs, but for now let's take the easy
16500          * road.
16501          */
16502         return IS_GEN(dev_priv, 6) &&
16503                 crtc_state->base.active &&
16504                 crtc_state->shared_dpll &&
16505                 crtc_state->port_clock == 0;
16506 }
16507
16508 static void intel_sanitize_encoder(struct intel_encoder *encoder)
16509 {
16510         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
16511         struct intel_connector *connector;
16512         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
16513         struct intel_crtc_state *crtc_state = crtc ?
16514                 to_intel_crtc_state(crtc->base.state) : NULL;
16515
16516         /* We need to check both for a crtc link (meaning that the
16517          * encoder is active and trying to read from a pipe) and the
16518          * pipe itself being active. */
16519         bool has_active_crtc = crtc_state &&
16520                 crtc_state->base.active;
16521
16522         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
16523                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
16524                               pipe_name(crtc->pipe));
16525                 has_active_crtc = false;
16526         }
16527
16528         connector = intel_encoder_find_connector(encoder);
16529         if (connector && !has_active_crtc) {
16530                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
16531                               encoder->base.base.id,
16532                               encoder->base.name);
16533
16534                 /* Connector is active, but has no active pipe. This is
16535                  * fallout from our resume register restoring. Disable
16536                  * the encoder manually again. */
16537                 if (crtc_state) {
16538                         struct drm_encoder *best_encoder;
16539
16540                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
16541                                       encoder->base.base.id,
16542                                       encoder->base.name);
16543
16544                         /* avoid oopsing in case the hooks consult best_encoder */
16545                         best_encoder = connector->base.state->best_encoder;
16546                         connector->base.state->best_encoder = &encoder->base;
16547
16548                         if (encoder->disable)
16549                                 encoder->disable(encoder, crtc_state,
16550                                                  connector->base.state);
16551                         if (encoder->post_disable)
16552                                 encoder->post_disable(encoder, crtc_state,
16553                                                       connector->base.state);
16554
16555                         connector->base.state->best_encoder = best_encoder;
16556                 }
16557                 encoder->base.crtc = NULL;
16558
16559                 /* Inconsistent output/port/pipe state happens presumably due to
16560                  * a bug in one of the get_hw_state functions. Or someplace else
16561                  * in our code, like the register restore mess on resume. Clamp
16562                  * things to off as a safer default. */
16563
16564                 connector->base.dpms = DRM_MODE_DPMS_OFF;
16565                 connector->base.encoder = NULL;
16566         }
16567
16568         /* notify opregion of the sanitized encoder state */
16569         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
16570
16571         if (INTEL_GEN(dev_priv) >= 11)
16572                 icl_sanitize_encoder_pll_mapping(encoder);
16573 }
16574
16575 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
16576 {
16577         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
16578
16579         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
16580                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
16581                 i915_disable_vga(dev_priv);
16582         }
16583 }
16584
16585 void i915_redisable_vga(struct drm_i915_private *dev_priv)
16586 {
16587         intel_wakeref_t wakeref;
16588
16589         /*
16590          * This function can be called both from intel_modeset_setup_hw_state or
16591          * at a very early point in our resume sequence, where the power well
16592          * structures are not yet restored. Since this function is at a very
16593          * paranoid "someone might have enabled VGA while we were not looking"
16594          * level, just check if the power well is enabled instead of trying to
16595          * follow the "don't touch the power well if we don't need it" policy
16596          * the rest of the driver uses.
16597          */
16598         wakeref = intel_display_power_get_if_enabled(dev_priv,
16599                                                      POWER_DOMAIN_VGA);
16600         if (!wakeref)
16601                 return;
16602
16603         i915_redisable_vga_power_on(dev_priv);
16604
16605         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref);
16606 }
16607
16608 /* FIXME read out full plane state for all planes */
16609 static void readout_plane_state(struct drm_i915_private *dev_priv)
16610 {
16611         struct intel_plane *plane;
16612         struct intel_crtc *crtc;
16613
16614         for_each_intel_plane(&dev_priv->drm, plane) {
16615                 struct intel_plane_state *plane_state =
16616                         to_intel_plane_state(plane->base.state);
16617                 struct intel_crtc_state *crtc_state;
16618                 enum pipe pipe = PIPE_A;
16619                 bool visible;
16620
16621                 visible = plane->get_hw_state(plane, &pipe);
16622
16623                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16624                 crtc_state = to_intel_crtc_state(crtc->base.state);
16625
16626                 intel_set_plane_visible(crtc_state, plane_state, visible);
16627
16628                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
16629                               plane->base.base.id, plane->base.name,
16630                               enableddisabled(visible), pipe_name(pipe));
16631         }
16632
16633         for_each_intel_crtc(&dev_priv->drm, crtc) {
16634                 struct intel_crtc_state *crtc_state =
16635                         to_intel_crtc_state(crtc->base.state);
16636
16637                 fixup_active_planes(crtc_state);
16638         }
16639 }
16640
16641 static void intel_modeset_readout_hw_state(struct drm_device *dev)
16642 {
16643         struct drm_i915_private *dev_priv = to_i915(dev);
16644         enum pipe pipe;
16645         struct intel_crtc *crtc;
16646         struct intel_encoder *encoder;
16647         struct intel_connector *connector;
16648         struct drm_connector_list_iter conn_iter;
16649         int i;
16650
16651         dev_priv->active_pipes = 0;
16652
16653         for_each_intel_crtc(dev, crtc) {
16654                 struct intel_crtc_state *crtc_state =
16655                         to_intel_crtc_state(crtc->base.state);
16656
16657                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
16658                 memset(crtc_state, 0, sizeof(*crtc_state));
16659                 __drm_atomic_helper_crtc_reset(&crtc->base, &crtc_state->base);
16660
16661                 crtc_state->base.active = crtc_state->base.enable =
16662                         dev_priv->display.get_pipe_config(crtc, crtc_state);
16663
16664                 crtc->base.enabled = crtc_state->base.enable;
16665                 crtc->active = crtc_state->base.active;
16666
16667                 if (crtc_state->base.active)
16668                         dev_priv->active_pipes |= BIT(crtc->pipe);
16669
16670                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16671                               crtc->base.base.id, crtc->base.name,
16672                               enableddisabled(crtc_state->base.active));
16673         }
16674
16675         readout_plane_state(dev_priv);
16676
16677         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16678                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16679
16680                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
16681                                                         &pll->state.hw_state);
16682
16683                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
16684                     pll->info->id == DPLL_ID_EHL_DPLL4) {
16685                         pll->wakeref = intel_display_power_get(dev_priv,
16686                                                                POWER_DOMAIN_DPLL_DC_OFF);
16687                 }
16688
16689                 pll->state.crtc_mask = 0;
16690                 for_each_intel_crtc(dev, crtc) {
16691                         struct intel_crtc_state *crtc_state =
16692                                 to_intel_crtc_state(crtc->base.state);
16693
16694                         if (crtc_state->base.active &&
16695                             crtc_state->shared_dpll == pll)
16696                                 pll->state.crtc_mask |= 1 << crtc->pipe;
16697                 }
16698                 pll->active_mask = pll->state.crtc_mask;
16699
16700                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
16701                               pll->info->name, pll->state.crtc_mask, pll->on);
16702         }
16703
16704         for_each_intel_encoder(dev, encoder) {
16705                 pipe = 0;
16706
16707                 if (encoder->get_hw_state(encoder, &pipe)) {
16708                         struct intel_crtc_state *crtc_state;
16709
16710                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16711                         crtc_state = to_intel_crtc_state(crtc->base.state);
16712
16713                         encoder->base.crtc = &crtc->base;
16714                         encoder->get_config(encoder, crtc_state);
16715                 } else {
16716                         encoder->base.crtc = NULL;
16717                 }
16718
16719                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16720                               encoder->base.base.id, encoder->base.name,
16721                               enableddisabled(encoder->base.crtc),
16722                               pipe_name(pipe));
16723         }
16724
16725         drm_connector_list_iter_begin(dev, &conn_iter);
16726         for_each_intel_connector_iter(connector, &conn_iter) {
16727                 if (connector->get_hw_state(connector)) {
16728                         connector->base.dpms = DRM_MODE_DPMS_ON;
16729
16730                         encoder = connector->encoder;
16731                         connector->base.encoder = &encoder->base;
16732
16733                         if (encoder->base.crtc &&
16734                             encoder->base.crtc->state->active) {
16735                                 /*
16736                                  * This has to be done during hardware readout
16737                                  * because anything calling .crtc_disable may
16738                                  * rely on the connector_mask being accurate.
16739                                  */
16740                                 encoder->base.crtc->state->connector_mask |=
16741                                         drm_connector_mask(&connector->base);
16742                                 encoder->base.crtc->state->encoder_mask |=
16743                                         drm_encoder_mask(&encoder->base);
16744                         }
16745
16746                 } else {
16747                         connector->base.dpms = DRM_MODE_DPMS_OFF;
16748                         connector->base.encoder = NULL;
16749                 }
16750                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16751                               connector->base.base.id, connector->base.name,
16752                               enableddisabled(connector->base.encoder));
16753         }
16754         drm_connector_list_iter_end(&conn_iter);
16755
16756         for_each_intel_crtc(dev, crtc) {
16757                 struct intel_bw_state *bw_state =
16758                         to_intel_bw_state(dev_priv->bw_obj.state);
16759                 struct intel_crtc_state *crtc_state =
16760                         to_intel_crtc_state(crtc->base.state);
16761                 struct intel_plane *plane;
16762                 int min_cdclk = 0;
16763
16764                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
16765                 if (crtc_state->base.active) {
16766                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
16767                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
16768                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
16769                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
16770                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
16771
16772                         /*
16773                          * The initial mode needs to be set in order to keep
16774                          * the atomic core happy. It wants a valid mode if the
16775                          * crtc's enabled, so we do the above call.
16776                          *
16777                          * But we don't set all the derived state fully, hence
16778                          * set a flag to indicate that a full recalculation is
16779                          * needed on the next commit.
16780                          */
16781                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
16782
16783                         intel_crtc_compute_pixel_rate(crtc_state);
16784
16785                         if (dev_priv->display.modeset_calc_cdclk) {
16786                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
16787                                 if (WARN_ON(min_cdclk < 0))
16788                                         min_cdclk = 0;
16789                         }
16790
16791                         drm_calc_timestamping_constants(&crtc->base,
16792                                                         &crtc_state->base.adjusted_mode);
16793                         update_scanline_offset(crtc_state);
16794                 }
16795
16796                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
16797                 dev_priv->min_voltage_level[crtc->pipe] =
16798                         crtc_state->min_voltage_level;
16799
16800                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
16801                         const struct intel_plane_state *plane_state =
16802                                 to_intel_plane_state(plane->base.state);
16803
16804                         /*
16805                          * FIXME don't have the fb yet, so can't
16806                          * use intel_plane_data_rate() :(
16807                          */
16808                         if (plane_state->base.visible)
16809                                 crtc_state->data_rate[plane->id] =
16810                                         4 * crtc_state->pixel_rate;
16811                 }
16812
16813                 intel_bw_crtc_update(bw_state, crtc_state);
16814
16815                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
16816         }
16817 }
16818
16819 static void
16820 get_encoder_power_domains(struct drm_i915_private *dev_priv)
16821 {
16822         struct intel_encoder *encoder;
16823
16824         for_each_intel_encoder(&dev_priv->drm, encoder) {
16825                 struct intel_crtc_state *crtc_state;
16826
16827                 if (!encoder->get_power_domains)
16828                         continue;
16829
16830                 /*
16831                  * MST-primary and inactive encoders don't have a crtc state
16832                  * and neither of these require any power domain references.
16833                  */
16834                 if (!encoder->base.crtc)
16835                         continue;
16836
16837                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
16838                 encoder->get_power_domains(encoder, crtc_state);
16839         }
16840 }
16841
16842 static void intel_early_display_was(struct drm_i915_private *dev_priv)
16843 {
16844         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
16845         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
16846                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
16847                            DARBF_GATING_DIS);
16848
16849         if (IS_HASWELL(dev_priv)) {
16850                 /*
16851                  * WaRsPkgCStateDisplayPMReq:hsw
16852                  * System hang if this isn't done before disabling all planes!
16853                  */
16854                 I915_WRITE(CHICKEN_PAR1_1,
16855                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
16856         }
16857 }
16858
16859 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
16860                                        enum port port, i915_reg_t hdmi_reg)
16861 {
16862         u32 val = I915_READ(hdmi_reg);
16863
16864         if (val & SDVO_ENABLE ||
16865             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
16866                 return;
16867
16868         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
16869                       port_name(port));
16870
16871         val &= ~SDVO_PIPE_SEL_MASK;
16872         val |= SDVO_PIPE_SEL(PIPE_A);
16873
16874         I915_WRITE(hdmi_reg, val);
16875 }
16876
16877 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
16878                                      enum port port, i915_reg_t dp_reg)
16879 {
16880         u32 val = I915_READ(dp_reg);
16881
16882         if (val & DP_PORT_EN ||
16883             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
16884                 return;
16885
16886         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
16887                       port_name(port));
16888
16889         val &= ~DP_PIPE_SEL_MASK;
16890         val |= DP_PIPE_SEL(PIPE_A);
16891
16892         I915_WRITE(dp_reg, val);
16893 }
16894
16895 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
16896 {
16897         /*
16898          * The BIOS may select transcoder B on some of the PCH
16899          * ports even it doesn't enable the port. This would trip
16900          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
16901          * Sanitize the transcoder select bits to prevent that. We
16902          * assume that the BIOS never actually enabled the port,
16903          * because if it did we'd actually have to toggle the port
16904          * on and back off to make the transcoder A select stick
16905          * (see. intel_dp_link_down(), intel_disable_hdmi(),
16906          * intel_disable_sdvo()).
16907          */
16908         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
16909         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
16910         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
16911
16912         /* PCH SDVOB multiplex with HDMIB */
16913         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
16914         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
16915         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
16916 }
16917
16918 /* Scan out the current hw modeset state,
16919  * and sanitizes it to the current state
16920  */
16921 static void
16922 intel_modeset_setup_hw_state(struct drm_device *dev,
16923                              struct drm_modeset_acquire_ctx *ctx)
16924 {
16925         struct drm_i915_private *dev_priv = to_i915(dev);
16926         struct intel_crtc_state *crtc_state;
16927         struct intel_encoder *encoder;
16928         struct intel_crtc *crtc;
16929         intel_wakeref_t wakeref;
16930         int i;
16931
16932         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
16933
16934         intel_early_display_was(dev_priv);
16935         intel_modeset_readout_hw_state(dev);
16936
16937         /* HW state is read out, now we need to sanitize this mess. */
16938
16939         /* Sanitize the TypeC port mode upfront, encoders depend on this */
16940         for_each_intel_encoder(dev, encoder) {
16941                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
16942
16943                 /* We need to sanitize only the MST primary port. */
16944                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
16945                     intel_phy_is_tc(dev_priv, phy))
16946                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
16947         }
16948
16949         get_encoder_power_domains(dev_priv);
16950
16951         if (HAS_PCH_IBX(dev_priv))
16952                 ibx_sanitize_pch_ports(dev_priv);
16953
16954         /*
16955          * intel_sanitize_plane_mapping() may need to do vblank
16956          * waits, so we need vblank interrupts restored beforehand.
16957          */
16958         for_each_intel_crtc(&dev_priv->drm, crtc) {
16959                 crtc_state = to_intel_crtc_state(crtc->base.state);
16960
16961                 drm_crtc_vblank_reset(&crtc->base);
16962
16963                 if (crtc_state->base.active)
16964                         intel_crtc_vblank_on(crtc_state);
16965         }
16966
16967         intel_sanitize_plane_mapping(dev_priv);
16968
16969         for_each_intel_encoder(dev, encoder)
16970                 intel_sanitize_encoder(encoder);
16971
16972         for_each_intel_crtc(&dev_priv->drm, crtc) {
16973                 crtc_state = to_intel_crtc_state(crtc->base.state);
16974                 intel_sanitize_crtc(crtc, ctx);
16975                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
16976         }
16977
16978         intel_modeset_update_connector_atomic_state(dev);
16979
16980         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16981                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16982
16983                 if (!pll->on || pll->active_mask)
16984                         continue;
16985
16986                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
16987                               pll->info->name);
16988
16989                 pll->info->funcs->disable(dev_priv, pll);
16990                 pll->on = false;
16991         }
16992
16993         if (IS_G4X(dev_priv)) {
16994                 g4x_wm_get_hw_state(dev_priv);
16995                 g4x_wm_sanitize(dev_priv);
16996         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16997                 vlv_wm_get_hw_state(dev_priv);
16998                 vlv_wm_sanitize(dev_priv);
16999         } else if (INTEL_GEN(dev_priv) >= 9) {
17000                 skl_wm_get_hw_state(dev_priv);
17001         } else if (HAS_PCH_SPLIT(dev_priv)) {
17002                 ilk_wm_get_hw_state(dev_priv);
17003         }
17004
17005         for_each_intel_crtc(dev, crtc) {
17006                 u64 put_domains;
17007
17008                 crtc_state = to_intel_crtc_state(crtc->base.state);
17009                 put_domains = modeset_get_crtc_power_domains(crtc_state);
17010                 if (WARN_ON(put_domains))
17011                         modeset_put_power_domains(dev_priv, put_domains);
17012         }
17013
17014         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
17015
17016         intel_fbc_init_pipe_state(dev_priv);
17017 }
17018
17019 void intel_display_resume(struct drm_device *dev)
17020 {
17021         struct drm_i915_private *dev_priv = to_i915(dev);
17022         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
17023         struct drm_modeset_acquire_ctx ctx;
17024         int ret;
17025
17026         dev_priv->modeset_restore_state = NULL;
17027         if (state)
17028                 state->acquire_ctx = &ctx;
17029
17030         drm_modeset_acquire_init(&ctx, 0);
17031
17032         while (1) {
17033                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
17034                 if (ret != -EDEADLK)
17035                         break;
17036
17037                 drm_modeset_backoff(&ctx);
17038         }
17039
17040         if (!ret)
17041                 ret = __intel_display_resume(dev, state, &ctx);
17042
17043         intel_enable_ipc(dev_priv);
17044         drm_modeset_drop_locks(&ctx);
17045         drm_modeset_acquire_fini(&ctx);
17046
17047         if (ret)
17048                 DRM_ERROR("Restoring old state failed with %i\n", ret);
17049         if (state)
17050                 drm_atomic_state_put(state);
17051 }
17052
17053 static void intel_hpd_poll_fini(struct drm_device *dev)
17054 {
17055         struct intel_connector *connector;
17056         struct drm_connector_list_iter conn_iter;
17057
17058         /* Kill all the work that may have been queued by hpd. */
17059         drm_connector_list_iter_begin(dev, &conn_iter);
17060         for_each_intel_connector_iter(connector, &conn_iter) {
17061                 if (connector->modeset_retry_work.func)
17062                         cancel_work_sync(&connector->modeset_retry_work);
17063                 if (connector->hdcp.shim) {
17064                         cancel_delayed_work_sync(&connector->hdcp.check_work);
17065                         cancel_work_sync(&connector->hdcp.prop_work);
17066                 }
17067         }
17068         drm_connector_list_iter_end(&conn_iter);
17069 }
17070
17071 void intel_modeset_driver_remove(struct drm_device *dev)
17072 {
17073         struct drm_i915_private *dev_priv = to_i915(dev);
17074
17075         flush_workqueue(dev_priv->modeset_wq);
17076
17077         flush_work(&dev_priv->atomic_helper.free_work);
17078         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
17079
17080         /*
17081          * Interrupts and polling as the first thing to avoid creating havoc.
17082          * Too much stuff here (turning of connectors, ...) would
17083          * experience fancy races otherwise.
17084          */
17085         intel_irq_uninstall(dev_priv);
17086
17087         /*
17088          * Due to the hpd irq storm handling the hotplug work can re-arm the
17089          * poll handlers. Hence disable polling after hpd handling is shut down.
17090          */
17091         intel_hpd_poll_fini(dev);
17092
17093         /* poll work can call into fbdev, hence clean that up afterwards */
17094         intel_fbdev_fini(dev_priv);
17095
17096         intel_unregister_dsm_handler();
17097
17098         intel_fbc_global_disable(dev_priv);
17099
17100         /* flush any delayed tasks or pending work */
17101         flush_scheduled_work();
17102
17103         intel_hdcp_component_fini(dev_priv);
17104
17105         drm_mode_config_cleanup(dev);
17106
17107         intel_overlay_cleanup(dev_priv);
17108
17109         intel_gmbus_teardown(dev_priv);
17110
17111         destroy_workqueue(dev_priv->modeset_wq);
17112
17113         intel_fbc_cleanup_cfb(dev_priv);
17114 }
17115
17116 /*
17117  * set vga decode state - true == enable VGA decode
17118  */
17119 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
17120 {
17121         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17122         u16 gmch_ctrl;
17123
17124         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
17125                 DRM_ERROR("failed to read control word\n");
17126                 return -EIO;
17127         }
17128
17129         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
17130                 return 0;
17131
17132         if (state)
17133                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
17134         else
17135                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
17136
17137         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
17138                 DRM_ERROR("failed to write control word\n");
17139                 return -EIO;
17140         }
17141
17142         return 0;
17143 }
17144
17145 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
17146
17147 struct intel_display_error_state {
17148
17149         u32 power_well_driver;
17150
17151         struct intel_cursor_error_state {
17152                 u32 control;
17153                 u32 position;
17154                 u32 base;
17155                 u32 size;
17156         } cursor[I915_MAX_PIPES];
17157
17158         struct intel_pipe_error_state {
17159                 bool power_domain_on;
17160                 u32 source;
17161                 u32 stat;
17162         } pipe[I915_MAX_PIPES];
17163
17164         struct intel_plane_error_state {
17165                 u32 control;
17166                 u32 stride;
17167                 u32 size;
17168                 u32 pos;
17169                 u32 addr;
17170                 u32 surface;
17171                 u32 tile_offset;
17172         } plane[I915_MAX_PIPES];
17173
17174         struct intel_transcoder_error_state {
17175                 bool available;
17176                 bool power_domain_on;
17177                 enum transcoder cpu_transcoder;
17178
17179                 u32 conf;
17180
17181                 u32 htotal;
17182                 u32 hblank;
17183                 u32 hsync;
17184                 u32 vtotal;
17185                 u32 vblank;
17186                 u32 vsync;
17187         } transcoder[5];
17188 };
17189
17190 struct intel_display_error_state *
17191 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
17192 {
17193         struct intel_display_error_state *error;
17194         int transcoders[] = {
17195                 TRANSCODER_A,
17196                 TRANSCODER_B,
17197                 TRANSCODER_C,
17198                 TRANSCODER_D,
17199                 TRANSCODER_EDP,
17200         };
17201         int i;
17202
17203         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
17204
17205         if (!HAS_DISPLAY(dev_priv))
17206                 return NULL;
17207
17208         error = kzalloc(sizeof(*error), GFP_ATOMIC);
17209         if (error == NULL)
17210                 return NULL;
17211
17212         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17213                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
17214
17215         for_each_pipe(dev_priv, i) {
17216                 error->pipe[i].power_domain_on =
17217                         __intel_display_power_is_enabled(dev_priv,
17218                                                          POWER_DOMAIN_PIPE(i));
17219                 if (!error->pipe[i].power_domain_on)
17220                         continue;
17221
17222                 error->cursor[i].control = I915_READ(CURCNTR(i));
17223                 error->cursor[i].position = I915_READ(CURPOS(i));
17224                 error->cursor[i].base = I915_READ(CURBASE(i));
17225
17226                 error->plane[i].control = I915_READ(DSPCNTR(i));
17227                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
17228                 if (INTEL_GEN(dev_priv) <= 3) {
17229                         error->plane[i].size = I915_READ(DSPSIZE(i));
17230                         error->plane[i].pos = I915_READ(DSPPOS(i));
17231                 }
17232                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17233                         error->plane[i].addr = I915_READ(DSPADDR(i));
17234                 if (INTEL_GEN(dev_priv) >= 4) {
17235                         error->plane[i].surface = I915_READ(DSPSURF(i));
17236                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
17237                 }
17238
17239                 error->pipe[i].source = I915_READ(PIPESRC(i));
17240
17241                 if (HAS_GMCH(dev_priv))
17242                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
17243         }
17244
17245         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17246                 enum transcoder cpu_transcoder = transcoders[i];
17247
17248                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
17249                         continue;
17250
17251                 error->transcoder[i].available = true;
17252                 error->transcoder[i].power_domain_on =
17253                         __intel_display_power_is_enabled(dev_priv,
17254                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
17255                 if (!error->transcoder[i].power_domain_on)
17256                         continue;
17257
17258                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
17259
17260                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
17261                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
17262                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
17263                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
17264                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
17265                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
17266                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
17267         }
17268
17269         return error;
17270 }
17271
17272 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
17273
17274 void
17275 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17276                                 struct intel_display_error_state *error)
17277 {
17278         struct drm_i915_private *dev_priv = m->i915;
17279         int i;
17280
17281         if (!error)
17282                 return;
17283
17284         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
17285         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17286                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
17287                            error->power_well_driver);
17288         for_each_pipe(dev_priv, i) {
17289                 err_printf(m, "Pipe [%d]:\n", i);
17290                 err_printf(m, "  Power: %s\n",
17291                            onoff(error->pipe[i].power_domain_on));
17292                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
17293                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
17294
17295                 err_printf(m, "Plane [%d]:\n", i);
17296                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
17297                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
17298                 if (INTEL_GEN(dev_priv) <= 3) {
17299                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
17300                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
17301                 }
17302                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17303                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
17304                 if (INTEL_GEN(dev_priv) >= 4) {
17305                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
17306                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
17307                 }
17308
17309                 err_printf(m, "Cursor [%d]:\n", i);
17310                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
17311                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
17312                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
17313         }
17314
17315         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17316                 if (!error->transcoder[i].available)
17317                         continue;
17318
17319                 err_printf(m, "CPU transcoder: %s\n",
17320                            transcoder_name(error->transcoder[i].cpu_transcoder));
17321                 err_printf(m, "  Power: %s\n",
17322                            onoff(error->transcoder[i].power_domain_on));
17323                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
17324                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
17325                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
17326                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
17327                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
17328                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
17329                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
17330         }
17331 }
17332
17333 #endif