drivers/mmc/host/sdhci-tegra.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2010 Google, Inc.
   4  */
   5
   6 #include <linux/delay.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/err.h>
   9 #include <linux/module.h>
  10 #include <linux/init.h>
  11 #include <linux/iopoll.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/clk.h>
  14 #include <linux/io.h>
  15 #include <linux/of.h>
  16 #include <linux/of_device.h>
  17 #include <linux/pinctrl/consumer.h>
  18 #include <linux/regulator/consumer.h>
  19 #include <linux/reset.h>
  20 #include <linux/mmc/card.h>
  21 #include <linux/mmc/host.h>
  22 #include <linux/mmc/mmc.h>
  23 #include <linux/mmc/slot-gpio.h>
  24 #include <linux/gpio/consumer.h>
  25 #include <linux/ktime.h>
  26
  27 #include "sdhci-pltfm.h"
  28 #include "cqhci.h"
  29
  30 /* Tegra SDHOST controller vendor register definitions */
  31 #define SDHCI_TEGRA_VENDOR_CLOCK_CTRL                   0x100
  32 #define SDHCI_CLOCK_CTRL_TAP_MASK                       0x00ff0000
  33 #define SDHCI_CLOCK_CTRL_TAP_SHIFT                      16
  34 #define SDHCI_CLOCK_CTRL_TRIM_MASK                      0x1f000000
  35 #define SDHCI_CLOCK_CTRL_TRIM_SHIFT                     24
  36 #define SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE          BIT(5)
  37 #define SDHCI_CLOCK_CTRL_PADPIPE_CLKEN_OVERRIDE         BIT(3)
  38 #define SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE        BIT(2)
  39
  40 #define SDHCI_TEGRA_VENDOR_SYS_SW_CTRL                  0x104
  41 #define SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE         BIT(31)
  42
  43 #define SDHCI_TEGRA_VENDOR_CAP_OVERRIDES                0x10c
  44 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK         0x00003f00
  45 #define SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT        8
  46
  47 #define SDHCI_TEGRA_VENDOR_MISC_CTRL                    0x120
  48 #define SDHCI_MISC_CTRL_ENABLE_SDR104                   0x8
  49 #define SDHCI_MISC_CTRL_ENABLE_SDR50                    0x10
  50 #define SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300           0x20
  51 #define SDHCI_MISC_CTRL_ENABLE_DDR50                    0x200
  52
  53 #define SDHCI_TEGRA_VENDOR_DLLCAL_CFG                   0x1b0
  54 #define SDHCI_TEGRA_DLLCAL_CALIBRATE                    BIT(31)
  55
  56 #define SDHCI_TEGRA_VENDOR_DLLCAL_STA                   0x1bc
  57 #define SDHCI_TEGRA_DLLCAL_STA_ACTIVE                   BIT(31)
  58
  59 #define SDHCI_VNDR_TUN_CTRL0_0                          0x1c0
  60 #define SDHCI_VNDR_TUN_CTRL0_TUN_HW_TAP                 0x20000
  61 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK         0x03fc0000
  62 #define SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT        18
  63 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK                 0x00001fc0
  64 #define SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT                6
  65 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK              0x000e000
  66 #define SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT             13
  67 #define TRIES_128                                       2
  68 #define TRIES_256                                       4
  69 #define SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK          0x7
  70
  71 #define SDHCI_TEGRA_VNDR_TUN_CTRL1_0                    0x1c4
  72 #define SDHCI_TEGRA_VNDR_TUN_STATUS0                    0x1C8
  73 #define SDHCI_TEGRA_VNDR_TUN_STATUS1                    0x1CC
  74 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK           0xFF
  75 #define SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT      0x8
  76 #define TUNING_WORD_BIT_SIZE                            32
  77
  78 #define SDHCI_TEGRA_AUTO_CAL_CONFIG                     0x1e4
  79 #define SDHCI_AUTO_CAL_START                            BIT(31)
  80 #define SDHCI_AUTO_CAL_ENABLE                           BIT(29)
  81 #define SDHCI_AUTO_CAL_PDPU_OFFSET_MASK                 0x0000ffff
  82
  83 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL                  0x1e0
  84 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK    0x0000000f
  85 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL     0x7
  86 #define SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD   BIT(31)
  87 #define SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK          0x07FFF000
  88
  89 #define SDHCI_TEGRA_AUTO_CAL_STATUS                     0x1ec
  90 #define SDHCI_TEGRA_AUTO_CAL_ACTIVE                     BIT(31)
  91
  92 #define NVQUIRK_FORCE_SDHCI_SPEC_200                    BIT(0)
  93 #define NVQUIRK_ENABLE_BLOCK_GAP_DET                    BIT(1)
  94 #define NVQUIRK_ENABLE_SDHCI_SPEC_300                   BIT(2)
  95 #define NVQUIRK_ENABLE_SDR50                            BIT(3)
  96 #define NVQUIRK_ENABLE_SDR104                           BIT(4)
  97 #define NVQUIRK_ENABLE_DDR50                            BIT(5)
  98 #define NVQUIRK_HAS_PADCALIB                            BIT(6)
  99 #define NVQUIRK_NEEDS_PAD_CONTROL                       BIT(7)
 100 #define NVQUIRK_DIS_CARD_CLK_CONFIG_TAP                 BIT(8)
 101 #define NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING               BIT(9)
 102
 103 /* SDMMC CQE Base Address for Tegra Host Ver 4.1 and Higher */
 104 #define SDHCI_TEGRA_CQE_BASE_ADDR                       0xF000
 105
 106 struct sdhci_tegra_soc_data {
 107         const struct sdhci_pltfm_data *pdata;
 108         u64 dma_mask;
 109         u32 nvquirks;
 110         u8 min_tap_delay;
 111         u8 max_tap_delay;
 112 };
 113
 114 /* Magic pull up and pull down pad calibration offsets */
 115 struct sdhci_tegra_autocal_offsets {
 116         u32 pull_up_3v3;
 117         u32 pull_down_3v3;
 118         u32 pull_up_3v3_timeout;
 119         u32 pull_down_3v3_timeout;
 120         u32 pull_up_1v8;
 121         u32 pull_down_1v8;
 122         u32 pull_up_1v8_timeout;
 123         u32 pull_down_1v8_timeout;
 124         u32 pull_up_sdr104;
 125         u32 pull_down_sdr104;
 126         u32 pull_up_hs400;
 127         u32 pull_down_hs400;
 128 };
 129
 130 struct sdhci_tegra {
 131         const struct sdhci_tegra_soc_data *soc_data;
 132         struct gpio_desc *power_gpio;
 133         bool ddr_signaling;
 134         bool pad_calib_required;
 135         bool pad_control_available;
 136
 137         struct reset_control *rst;
 138         struct pinctrl *pinctrl_sdmmc;
 139         struct pinctrl_state *pinctrl_state_3v3;
 140         struct pinctrl_state *pinctrl_state_1v8;
 141         struct pinctrl_state *pinctrl_state_3v3_drv;
 142         struct pinctrl_state *pinctrl_state_1v8_drv;
 143
 144         struct sdhci_tegra_autocal_offsets autocal_offsets;
 145         ktime_t last_calib;
 146
 147         u32 default_tap;
 148         u32 default_trim;
 149         u32 dqs_trim;
 150         bool enable_hwcq;
 151         unsigned long curr_clk_rate;
 152         u8 tuned_tap_delay;
 153 };
 154
 155 static u16 tegra_sdhci_readw(struct sdhci_host *host, int reg)
 156 {
 157         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 158         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 159         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 160
 161         if (unlikely((soc_data->nvquirks & NVQUIRK_FORCE_SDHCI_SPEC_200) &&
 162                         (reg == SDHCI_HOST_VERSION))) {
 163                 /* Erratum: Version register is invalid in HW. */
 164                 return SDHCI_SPEC_200;
 165         }
 166
 167         return readw(host->ioaddr + reg);
 168 }
 169
 170 static void tegra_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 171 {
 172         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 173
 174         switch (reg) {
 175         case SDHCI_TRANSFER_MODE:
 176                 /*
 177                  * Postpone this write, we must do it together with a
 178                  * command write that is down below.
 179                  */
 180                 pltfm_host->xfer_mode_shadow = val;
 181                 return;
 182         case SDHCI_COMMAND:
 183                 writel((val << 16) | pltfm_host->xfer_mode_shadow,
 184                         host->ioaddr + SDHCI_TRANSFER_MODE);
 185                 return;
 186         }
 187
 188         writew(val, host->ioaddr + reg);
 189 }
 190
 191 static void tegra_sdhci_writel(struct sdhci_host *host, u32 val, int reg)
 192 {
 193         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 194         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 195         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 196
 197         /* Seems like we're getting spurious timeout and crc errors, so
 198          * disable signalling of them. In case of real errors software
 199          * timers should take care of eventually detecting them.
 200          */
 201         if (unlikely(reg == SDHCI_SIGNAL_ENABLE))
 202                 val &= ~(SDHCI_INT_TIMEOUT|SDHCI_INT_CRC);
 203
 204         writel(val, host->ioaddr + reg);
 205
 206         if (unlikely((soc_data->nvquirks & NVQUIRK_ENABLE_BLOCK_GAP_DET) &&
 207                         (reg == SDHCI_INT_ENABLE))) {
 208                 /* Erratum: Must enable block gap interrupt detection */
 209                 u8 gap_ctrl = readb(host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 210                 if (val & SDHCI_INT_CARD_INT)
 211                         gap_ctrl |= 0x8;
 212                 else
 213                         gap_ctrl &= ~0x8;
 214                 writeb(gap_ctrl, host->ioaddr + SDHCI_BLOCK_GAP_CONTROL);
 215         }
 216 }
 217
 218 static bool tegra_sdhci_configure_card_clk(struct sdhci_host *host, bool enable)
 219 {
 220         bool status;
 221         u32 reg;
 222
 223         reg = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
 224         status = !!(reg & SDHCI_CLOCK_CARD_EN);
 225
 226         if (status == enable)
 227                 return status;
 228
 229         if (enable)
 230                 reg |= SDHCI_CLOCK_CARD_EN;
 231         else
 232                 reg &= ~SDHCI_CLOCK_CARD_EN;
 233
 234         sdhci_writew(host, reg, SDHCI_CLOCK_CONTROL);
 235
 236         return status;
 237 }
 238
 239 static void tegra210_sdhci_writew(struct sdhci_host *host, u16 val, int reg)
 240 {
 241         bool is_tuning_cmd = 0;
 242         bool clk_enabled;
 243         u8 cmd;
 244
 245         if (reg == SDHCI_COMMAND) {
 246                 cmd = SDHCI_GET_CMD(val);
 247                 is_tuning_cmd = cmd == MMC_SEND_TUNING_BLOCK ||
 248                                 cmd == MMC_SEND_TUNING_BLOCK_HS200;
 249         }
 250
 251         if (is_tuning_cmd)
 252                 clk_enabled = tegra_sdhci_configure_card_clk(host, 0);
 253
 254         writew(val, host->ioaddr + reg);
 255
 256         if (is_tuning_cmd) {
 257                 udelay(1);
 258                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 259                 tegra_sdhci_configure_card_clk(host, clk_enabled);
 260         }
 261 }
 262
 263 static unsigned int tegra_sdhci_get_ro(struct sdhci_host *host)
 264 {
 265         /*
 266          * Write-enable shall be assumed if GPIO is missing in a board's
 267          * device-tree because SDHCI's WRITE_PROTECT bit doesn't work on
 268          * Tegra.
 269          */
 270         return mmc_gpio_get_ro(host->mmc);
 271 }
 272
 273 static bool tegra_sdhci_is_pad_and_regulator_valid(struct sdhci_host *host)
 274 {
 275         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 276         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 277         int has_1v8, has_3v3;
 278
 279         /*
 280          * The SoCs which have NVQUIRK_NEEDS_PAD_CONTROL require software pad
 281          * voltage configuration in order to perform voltage switching. This
 282          * means that valid pinctrl info is required on SDHCI instances capable
 283          * of performing voltage switching. Whether or not an SDHCI instance is
 284          * capable of voltage switching is determined based on the regulator.
 285          */
 286
 287         if (!(tegra_host->soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL))
 288                 return true;
 289
 290         if (IS_ERR(host->mmc->supply.vqmmc))
 291                 return false;
 292
 293         has_1v8 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 294                                                  1700000, 1950000);
 295
 296         has_3v3 = regulator_is_supported_voltage(host->mmc->supply.vqmmc,
 297                                                  2700000, 3600000);
 298
 299         if (has_1v8 == 1 && has_3v3 == 1)
 300                 return tegra_host->pad_control_available;
 301
 302         /* Fixed voltage, no pad control required. */
 303         return true;
 304 }
 305
 306 static void tegra_sdhci_set_tap(struct sdhci_host *host, unsigned int tap)
 307 {
 308         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 309         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 310         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 311         bool card_clk_enabled = false;
 312         u32 reg;
 313
 314         /*
 315          * Touching the tap values is a bit tricky on some SoC generations.
 316          * The quirk enables a workaround for a glitch that sometimes occurs if
 317          * the tap values are changed.
 318          */
 319
 320         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP)
 321                 card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 322
 323         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 324         reg &= ~SDHCI_CLOCK_CTRL_TAP_MASK;
 325         reg |= tap << SDHCI_CLOCK_CTRL_TAP_SHIFT;
 326         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 327
 328         if (soc_data->nvquirks & NVQUIRK_DIS_CARD_CLK_CONFIG_TAP &&
 329             card_clk_enabled) {
 330                 udelay(1);
 331                 sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 332                 tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 333         }
 334 }
 335
 336 static void tegra_sdhci_hs400_enhanced_strobe(struct mmc_host *mmc,
 337                                               struct mmc_ios *ios)
 338 {
 339         struct sdhci_host *host = mmc_priv(mmc);
 340         u32 val;
 341
 342         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 343
 344         if (ios->enhanced_strobe)
 345                 val |= SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 346         else
 347                 val &= ~SDHCI_TEGRA_SYS_SW_CTRL_ENHANCED_STROBE;
 348
 349         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_SYS_SW_CTRL);
 350
 351 }
 352
 353 static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask)
 354 {
 355         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 356         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 357         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 358         u32 misc_ctrl, clk_ctrl, pad_ctrl;
 359
 360         sdhci_reset(host, mask);
 361
 362         if (!(mask & SDHCI_RESET_ALL))
 363                 return;
 364
 365         tegra_sdhci_set_tap(host, tegra_host->default_tap);
 366
 367         misc_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 368         clk_ctrl = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 369
 370         misc_ctrl &= ~(SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300 |
 371                        SDHCI_MISC_CTRL_ENABLE_SDR50 |
 372                        SDHCI_MISC_CTRL_ENABLE_DDR50 |
 373                        SDHCI_MISC_CTRL_ENABLE_SDR104);
 374
 375         clk_ctrl &= ~(SDHCI_CLOCK_CTRL_TRIM_MASK |
 376                       SDHCI_CLOCK_CTRL_SPI_MODE_CLKEN_OVERRIDE);
 377
 378         if (tegra_sdhci_is_pad_and_regulator_valid(host)) {
 379                 /* Erratum: Enable SDHCI spec v3.00 support */
 380                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDHCI_SPEC_300)
 381                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDHCI_SPEC_300;
 382                 /* Advertise UHS modes as supported by host */
 383                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR50)
 384                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR50;
 385                 if (soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
 386                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_DDR50;
 387                 if (soc_data->nvquirks & NVQUIRK_ENABLE_SDR104)
 388                         misc_ctrl |= SDHCI_MISC_CTRL_ENABLE_SDR104;
 389                 if (soc_data->nvquirks & SDHCI_MISC_CTRL_ENABLE_SDR50)
 390                         clk_ctrl |= SDHCI_CLOCK_CTRL_SDR50_TUNING_OVERRIDE;
 391         }
 392
 393         clk_ctrl |= tegra_host->default_trim << SDHCI_CLOCK_CTRL_TRIM_SHIFT;
 394
 395         sdhci_writel(host, misc_ctrl, SDHCI_TEGRA_VENDOR_MISC_CTRL);
 396         sdhci_writel(host, clk_ctrl, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 397
 398         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB) {
 399                 pad_ctrl = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 400                 pad_ctrl &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_MASK;
 401                 pad_ctrl |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_VREF_SEL_VAL;
 402                 sdhci_writel(host, pad_ctrl, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 403
 404                 tegra_host->pad_calib_required = true;
 405         }
 406
 407         tegra_host->ddr_signaling = false;
 408 }
 409
 410 static void tegra_sdhci_configure_cal_pad(struct sdhci_host *host, bool enable)
 411 {
 412         u32 val;
 413
 414         /*
 415          * Enable or disable the additional I/O pad used by the drive strength
 416          * calibration process.
 417          */
 418         val = sdhci_readl(host, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 419
 420         if (enable)
 421                 val |= SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 422         else
 423                 val &= ~SDHCI_TEGRA_SDMEM_COMP_PADCTRL_E_INPUT_E_PWRD;
 424
 425         sdhci_writel(host, val, SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 426
 427         if (enable)
 428                 usleep_range(1, 2);
 429 }
 430
 431 static void tegra_sdhci_set_pad_autocal_offset(struct sdhci_host *host,
 432                                                u16 pdpu)
 433 {
 434         u32 reg;
 435
 436         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 437         reg &= ~SDHCI_AUTO_CAL_PDPU_OFFSET_MASK;
 438         reg |= pdpu;
 439         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 440 }
 441
 442 static int tegra_sdhci_set_padctrl(struct sdhci_host *host, int voltage,
 443                                    bool state_drvupdn)
 444 {
 445         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 446         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 447         struct sdhci_tegra_autocal_offsets *offsets =
 448                                                 &tegra_host->autocal_offsets;
 449         struct pinctrl_state *pinctrl_drvupdn = NULL;
 450         int ret = 0;
 451         u8 drvup = 0, drvdn = 0;
 452         u32 reg;
 453
 454         if (!state_drvupdn) {
 455                 /* PADS Drive Strength */
 456                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 457                         if (tegra_host->pinctrl_state_1v8_drv) {
 458                                 pinctrl_drvupdn =
 459                                         tegra_host->pinctrl_state_1v8_drv;
 460                         } else {
 461                                 drvup = offsets->pull_up_1v8_timeout;
 462                                 drvdn = offsets->pull_down_1v8_timeout;
 463                         }
 464                 } else {
 465                         if (tegra_host->pinctrl_state_3v3_drv) {
 466                                 pinctrl_drvupdn =
 467                                         tegra_host->pinctrl_state_3v3_drv;
 468                         } else {
 469                                 drvup = offsets->pull_up_3v3_timeout;
 470                                 drvdn = offsets->pull_down_3v3_timeout;
 471                         }
 472                 }
 473
 474                 if (pinctrl_drvupdn != NULL) {
 475                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 476                                                         pinctrl_drvupdn);
 477                         if (ret < 0)
 478                                 dev_err(mmc_dev(host->mmc),
 479                                         "failed pads drvupdn, ret: %d\n", ret);
 480                 } else if ((drvup) || (drvdn)) {
 481                         reg = sdhci_readl(host,
 482                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 483                         reg &= ~SDHCI_COMP_PADCTRL_DRVUPDN_OFFSET_MASK;
 484                         reg |= (drvup << 20) | (drvdn << 12);
 485                         sdhci_writel(host, reg,
 486                                         SDHCI_TEGRA_SDMEM_COMP_PADCTRL);
 487                 }
 488
 489         } else {
 490                 /* Dual Voltage PADS Voltage selection */
 491                 if (!tegra_host->pad_control_available)
 492                         return 0;
 493
 494                 if (voltage == MMC_SIGNAL_VOLTAGE_180) {
 495                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 496                                                 tegra_host->pinctrl_state_1v8);
 497                         if (ret < 0)
 498                                 dev_err(mmc_dev(host->mmc),
 499                                         "setting 1.8V failed, ret: %d\n", ret);
 500                 } else {
 501                         ret = pinctrl_select_state(tegra_host->pinctrl_sdmmc,
 502                                                 tegra_host->pinctrl_state_3v3);
 503                         if (ret < 0)
 504                                 dev_err(mmc_dev(host->mmc),
 505                                         "setting 3.3V failed, ret: %d\n", ret);
 506                 }
 507         }
 508
 509         return ret;
 510 }
 511
 512 static void tegra_sdhci_pad_autocalib(struct sdhci_host *host)
 513 {
 514         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 515         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 516         struct sdhci_tegra_autocal_offsets offsets =
 517                         tegra_host->autocal_offsets;
 518         struct mmc_ios *ios = &host->mmc->ios;
 519         bool card_clk_enabled;
 520         u16 pdpu;
 521         u32 reg;
 522         int ret;
 523
 524         switch (ios->timing) {
 525         case MMC_TIMING_UHS_SDR104:
 526                 pdpu = offsets.pull_down_sdr104 << 8 | offsets.pull_up_sdr104;
 527                 break;
 528         case MMC_TIMING_MMC_HS400:
 529                 pdpu = offsets.pull_down_hs400 << 8 | offsets.pull_up_hs400;
 530                 break;
 531         default:
 532                 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
 533                         pdpu = offsets.pull_down_1v8 << 8 | offsets.pull_up_1v8;
 534                 else
 535                         pdpu = offsets.pull_down_3v3 << 8 | offsets.pull_up_3v3;
 536         }
 537
 538         /* Set initial offset before auto-calibration */
 539         tegra_sdhci_set_pad_autocal_offset(host, pdpu);
 540
 541         card_clk_enabled = tegra_sdhci_configure_card_clk(host, false);
 542
 543         tegra_sdhci_configure_cal_pad(host, true);
 544
 545         reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 546         reg |= SDHCI_AUTO_CAL_ENABLE | SDHCI_AUTO_CAL_START;
 547         sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 548
 549         usleep_range(1, 2);
 550         /* 10 ms timeout */
 551         ret = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_AUTO_CAL_STATUS,
 552                                  reg, !(reg & SDHCI_TEGRA_AUTO_CAL_ACTIVE),
 553                                  1000, 10000);
 554
 555         tegra_sdhci_configure_cal_pad(host, false);
 556
 557         tegra_sdhci_configure_card_clk(host, card_clk_enabled);
 558
 559         if (ret) {
 560                 dev_err(mmc_dev(host->mmc), "Pad autocal timed out\n");
 561
 562                 /* Disable automatic cal and use fixed Drive Strengths */
 563                 reg = sdhci_readl(host, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 564                 reg &= ~SDHCI_AUTO_CAL_ENABLE;
 565                 sdhci_writel(host, reg, SDHCI_TEGRA_AUTO_CAL_CONFIG);
 566
 567                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, false);
 568                 if (ret < 0)
 569                         dev_err(mmc_dev(host->mmc),
 570                                 "Setting drive strengths failed: %d\n", ret);
 571         }
 572 }
 573
 574 static void tegra_sdhci_parse_pad_autocal_dt(struct sdhci_host *host)
 575 {
 576         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 577         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 578         struct sdhci_tegra_autocal_offsets *autocal =
 579                         &tegra_host->autocal_offsets;
 580         int err;
 581
 582         err = device_property_read_u32(host->mmc->parent,
 583                         "nvidia,pad-autocal-pull-up-offset-3v3",
 584                         &autocal->pull_up_3v3);
 585         if (err)
 586                 autocal->pull_up_3v3 = 0;
 587
 588         err = device_property_read_u32(host->mmc->parent,
 589                         "nvidia,pad-autocal-pull-down-offset-3v3",
 590                         &autocal->pull_down_3v3);
 591         if (err)
 592                 autocal->pull_down_3v3 = 0;
 593
 594         err = device_property_read_u32(host->mmc->parent,
 595                         "nvidia,pad-autocal-pull-up-offset-1v8",
 596                         &autocal->pull_up_1v8);
 597         if (err)
 598                 autocal->pull_up_1v8 = 0;
 599
 600         err = device_property_read_u32(host->mmc->parent,
 601                         "nvidia,pad-autocal-pull-down-offset-1v8",
 602                         &autocal->pull_down_1v8);
 603         if (err)
 604                 autocal->pull_down_1v8 = 0;
 605
 606         err = device_property_read_u32(host->mmc->parent,
 607                         "nvidia,pad-autocal-pull-up-offset-3v3-timeout",
 608                         &autocal->pull_up_3v3_timeout);
 609         if (err) {
 610                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 611                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 612                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 613                                 mmc_hostname(host->mmc));
 614                 autocal->pull_up_3v3_timeout = 0;
 615         }
 616
 617         err = device_property_read_u32(host->mmc->parent,
 618                         "nvidia,pad-autocal-pull-down-offset-3v3-timeout",
 619                         &autocal->pull_down_3v3_timeout);
 620         if (err) {
 621                 if (!IS_ERR(tegra_host->pinctrl_state_3v3) &&
 622                         (tegra_host->pinctrl_state_3v3_drv == NULL))
 623                         pr_warn("%s: Missing autocal timeout 3v3-pad drvs\n",
 624                                 mmc_hostname(host->mmc));
 625                 autocal->pull_down_3v3_timeout = 0;
 626         }
 627
 628         err = device_property_read_u32(host->mmc->parent,
 629                         "nvidia,pad-autocal-pull-up-offset-1v8-timeout",
 630                         &autocal->pull_up_1v8_timeout);
 631         if (err) {
 632                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 633                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 634                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 635                                 mmc_hostname(host->mmc));
 636                 autocal->pull_up_1v8_timeout = 0;
 637         }
 638
 639         err = device_property_read_u32(host->mmc->parent,
 640                         "nvidia,pad-autocal-pull-down-offset-1v8-timeout",
 641                         &autocal->pull_down_1v8_timeout);
 642         if (err) {
 643                 if (!IS_ERR(tegra_host->pinctrl_state_1v8) &&
 644                         (tegra_host->pinctrl_state_1v8_drv == NULL))
 645                         pr_warn("%s: Missing autocal timeout 1v8-pad drvs\n",
 646                                 mmc_hostname(host->mmc));
 647                 autocal->pull_down_1v8_timeout = 0;
 648         }
 649
 650         err = device_property_read_u32(host->mmc->parent,
 651                         "nvidia,pad-autocal-pull-up-offset-sdr104",
 652                         &autocal->pull_up_sdr104);
 653         if (err)
 654                 autocal->pull_up_sdr104 = autocal->pull_up_1v8;
 655
 656         err = device_property_read_u32(host->mmc->parent,
 657                         "nvidia,pad-autocal-pull-down-offset-sdr104",
 658                         &autocal->pull_down_sdr104);
 659         if (err)
 660                 autocal->pull_down_sdr104 = autocal->pull_down_1v8;
 661
 662         err = device_property_read_u32(host->mmc->parent,
 663                         "nvidia,pad-autocal-pull-up-offset-hs400",
 664                         &autocal->pull_up_hs400);
 665         if (err)
 666                 autocal->pull_up_hs400 = autocal->pull_up_1v8;
 667
 668         err = device_property_read_u32(host->mmc->parent,
 669                         "nvidia,pad-autocal-pull-down-offset-hs400",
 670                         &autocal->pull_down_hs400);
 671         if (err)
 672                 autocal->pull_down_hs400 = autocal->pull_down_1v8;
 673 }
 674
 675 static void tegra_sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 676 {
 677         struct sdhci_host *host = mmc_priv(mmc);
 678         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 679         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 680         ktime_t since_calib = ktime_sub(ktime_get(), tegra_host->last_calib);
 681
 682         /* 100 ms calibration interval is specified in the TRM */
 683         if (ktime_to_ms(since_calib) > 100) {
 684                 tegra_sdhci_pad_autocalib(host);
 685                 tegra_host->last_calib = ktime_get();
 686         }
 687
 688         sdhci_request(mmc, mrq);
 689 }
 690
 691 static void tegra_sdhci_parse_tap_and_trim(struct sdhci_host *host)
 692 {
 693         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 694         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 695         int err;
 696
 697         err = device_property_read_u32(host->mmc->parent, "nvidia,default-tap",
 698                                        &tegra_host->default_tap);
 699         if (err)
 700                 tegra_host->default_tap = 0;
 701
 702         err = device_property_read_u32(host->mmc->parent, "nvidia,default-trim",
 703                                        &tegra_host->default_trim);
 704         if (err)
 705                 tegra_host->default_trim = 0;
 706
 707         err = device_property_read_u32(host->mmc->parent, "nvidia,dqs-trim",
 708                                        &tegra_host->dqs_trim);
 709         if (err)
 710                 tegra_host->dqs_trim = 0x11;
 711 }
 712
 713 static void tegra_sdhci_parse_dt(struct sdhci_host *host)
 714 {
 715         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 716         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 717
 718         if (device_property_read_bool(host->mmc->parent, "supports-cqe"))
 719                 tegra_host->enable_hwcq = true;
 720         else
 721                 tegra_host->enable_hwcq = false;
 722
 723         tegra_sdhci_parse_pad_autocal_dt(host);
 724         tegra_sdhci_parse_tap_and_trim(host);
 725 }
 726
 727 static void tegra_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
 728 {
 729         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 730         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 731         unsigned long host_clk;
 732
 733         if (!clock)
 734                 return sdhci_set_clock(host, clock);
 735
 736         /*
 737          * In DDR50/52 modes the Tegra SDHCI controllers require the SDHCI
 738          * divider to be configured to divided the host clock by two. The SDHCI
 739          * clock divider is calculated as part of sdhci_set_clock() by
 740          * sdhci_calc_clk(). The divider is calculated from host->max_clk and
 741          * the requested clock rate.
 742          *
 743          * By setting the host->max_clk to clock * 2 the divider calculation
 744          * will always result in the correct value for DDR50/52 modes,
 745          * regardless of clock rate rounding, which may happen if the value
 746          * from clk_get_rate() is used.
 747          */
 748         host_clk = tegra_host->ddr_signaling ? clock * 2 : clock;
 749         clk_set_rate(pltfm_host->clk, host_clk);
 750         tegra_host->curr_clk_rate = host_clk;
 751         if (tegra_host->ddr_signaling)
 752                 host->max_clk = host_clk;
 753         else
 754                 host->max_clk = clk_get_rate(pltfm_host->clk);
 755
 756         sdhci_set_clock(host, clock);
 757
 758         if (tegra_host->pad_calib_required) {
 759                 tegra_sdhci_pad_autocalib(host);
 760                 tegra_host->pad_calib_required = false;
 761         }
 762 }
 763
 764 static unsigned int tegra_sdhci_get_max_clock(struct sdhci_host *host)
 765 {
 766         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 767
 768         return clk_round_rate(pltfm_host->clk, UINT_MAX);
 769 }
 770
 771 static void tegra_sdhci_set_dqs_trim(struct sdhci_host *host, u8 trim)
 772 {
 773         u32 val;
 774
 775         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 776         val &= ~SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_MASK;
 777         val |= trim << SDHCI_TEGRA_CAP_OVERRIDES_DQS_TRIM_SHIFT;
 778         sdhci_writel(host, val, SDHCI_TEGRA_VENDOR_CAP_OVERRIDES);
 779 }
 780
 781 static void tegra_sdhci_hs400_dll_cal(struct sdhci_host *host)
 782 {
 783         u32 reg;
 784         int err;
 785
 786         reg = sdhci_readl(host, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 787         reg |= SDHCI_TEGRA_DLLCAL_CALIBRATE;
 788         sdhci_writel(host, reg, SDHCI_TEGRA_VENDOR_DLLCAL_CFG);
 789
 790         /* 1 ms sleep, 5 ms timeout */
 791         err = readl_poll_timeout(host->ioaddr + SDHCI_TEGRA_VENDOR_DLLCAL_STA,
 792                                  reg, !(reg & SDHCI_TEGRA_DLLCAL_STA_ACTIVE),
 793                                  1000, 5000);
 794         if (err)
 795                 dev_err(mmc_dev(host->mmc),
 796                         "HS400 delay line calibration timed out\n");
 797 }
 798
 799 static void tegra_sdhci_tap_correction(struct sdhci_host *host, u8 thd_up,
 800                                        u8 thd_low, u8 fixed_tap)
 801 {
 802         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 803         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 804         u32 val, tun_status;
 805         u8 word, bit, edge1, tap, window;
 806         bool tap_result;
 807         bool start_fail = false;
 808         bool start_pass = false;
 809         bool end_pass = false;
 810         bool first_fail = false;
 811         bool first_pass = false;
 812         u8 start_pass_tap = 0;
 813         u8 end_pass_tap = 0;
 814         u8 first_fail_tap = 0;
 815         u8 first_pass_tap = 0;
 816         u8 total_tuning_words = host->tuning_loop_count / TUNING_WORD_BIT_SIZE;
 817
 818         /*
 819          * Read auto-tuned results and extract good valid passing window by
 820          * filtering out un-wanted bubble/partial/merged windows.
 821          */
 822         for (word = 0; word < total_tuning_words; word++) {
 823                 val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
 824                 val &= ~SDHCI_VNDR_TUN_CTRL0_TUN_WORD_SEL_MASK;
 825                 val |= word;
 826                 sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
 827                 tun_status = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS0);
 828                 bit = 0;
 829                 while (bit < TUNING_WORD_BIT_SIZE) {
 830                         tap = word * TUNING_WORD_BIT_SIZE + bit;
 831                         tap_result = tun_status & (1 << bit);
 832                         if (!tap_result && !start_fail) {
 833                                 start_fail = true;
 834                                 if (!first_fail) {
 835                                         first_fail_tap = tap;
 836                                         first_fail = true;
 837                                 }
 838
 839                         } else if (tap_result && start_fail && !start_pass) {
 840                                 start_pass_tap = tap;
 841                                 start_pass = true;
 842                                 if (!first_pass) {
 843                                         first_pass_tap = tap;
 844                                         first_pass = true;
 845                                 }
 846
 847                         } else if (!tap_result && start_fail && start_pass &&
 848                                    !end_pass) {
 849                                 end_pass_tap = tap - 1;
 850                                 end_pass = true;
 851                         } else if (tap_result && start_pass && start_fail &&
 852                                    end_pass) {
 853                                 window = end_pass_tap - start_pass_tap;
 854                                 /* discard merged window and bubble window */
 855                                 if (window >= thd_up || window < thd_low) {
 856                                         start_pass_tap = tap;
 857                                         end_pass = false;
 858                                 } else {
 859                                         /* set tap at middle of valid window */
 860                                         tap = start_pass_tap + window / 2;
 861                                         tegra_host->tuned_tap_delay = tap;
 862                                         return;
 863                                 }
 864                         }
 865
 866                         bit++;
 867                 }
 868         }
 869
 870         if (!first_fail) {
 871                 WARN(1, "no edge detected, continue with hw tuned delay.\n");
 872         } else if (first_pass) {
 873                 /* set tap location at fixed tap relative to the first edge */
 874                 edge1 = first_fail_tap + (first_pass_tap - first_fail_tap) / 2;
 875                 if (edge1 - 1 > fixed_tap)
 876                         tegra_host->tuned_tap_delay = edge1 - fixed_tap;
 877                 else
 878                         tegra_host->tuned_tap_delay = edge1 + fixed_tap;
 879         }
 880 }
 881
 882 static void tegra_sdhci_post_tuning(struct sdhci_host *host)
 883 {
 884         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 885         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 886         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
 887         u32 avg_tap_dly, val, min_tap_dly, max_tap_dly;
 888         u8 fixed_tap, start_tap, end_tap, window_width;
 889         u8 thdupper, thdlower;
 890         u8 num_iter;
 891         u32 clk_rate_mhz, period_ps, bestcase, worstcase;
 892
 893         /* retain HW tuned tap to use incase if no correction is needed */
 894         val = sdhci_readl(host, SDHCI_TEGRA_VENDOR_CLOCK_CTRL);
 895         tegra_host->tuned_tap_delay = (val & SDHCI_CLOCK_CTRL_TAP_MASK) >>
 896                                       SDHCI_CLOCK_CTRL_TAP_SHIFT;
 897         if (soc_data->min_tap_delay && soc_data->max_tap_delay) {
 898                 min_tap_dly = soc_data->min_tap_delay;
 899                 max_tap_dly = soc_data->max_tap_delay;
 900                 clk_rate_mhz = tegra_host->curr_clk_rate / USEC_PER_SEC;
 901                 period_ps = USEC_PER_SEC / clk_rate_mhz;
 902                 bestcase = period_ps / min_tap_dly;
 903                 worstcase = period_ps / max_tap_dly;
 904                 /*
 905                  * Upper and Lower bound thresholds used to detect merged and
 906                  * bubble windows
 907                  */
 908                 thdupper = (2 * worstcase + bestcase) / 2;
 909                 thdlower = worstcase / 4;
 910                 /*
 911                  * fixed tap is used when HW tuning result contains single edge
 912                  * and tap is set at fixed tap delay relative to the first edge
 913                  */
 914                 avg_tap_dly = (period_ps * 2) / (min_tap_dly + max_tap_dly);
 915                 fixed_tap = avg_tap_dly / 2;
 916
 917                 val = sdhci_readl(host, SDHCI_TEGRA_VNDR_TUN_STATUS1);
 918                 start_tap = val & SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 919                 end_tap = (val >> SDHCI_TEGRA_VNDR_TUN_STATUS1_END_TAP_SHIFT) &
 920                           SDHCI_TEGRA_VNDR_TUN_STATUS1_TAP_MASK;
 921                 window_width = end_tap - start_tap;
 922                 num_iter = host->tuning_loop_count;
 923                 /*
 924                  * partial window includes edges of the tuning range.
 925                  * merged window includes more taps so window width is higher
 926                  * than upper threshold.
 927                  */
 928                 if (start_tap == 0 || (end_tap == (num_iter - 1)) ||
 929                     (end_tap == num_iter - 2) || window_width >= thdupper) {
 930                         pr_debug("%s: Apply tuning correction\n",
 931                                  mmc_hostname(host->mmc));
 932                         tegra_sdhci_tap_correction(host, thdupper, thdlower,
 933                                                    fixed_tap);
 934                 }
 935         }
 936
 937         tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
 938 }
 939
 940 static int tegra_sdhci_execute_hw_tuning(struct mmc_host *mmc, u32 opcode)
 941 {
 942         struct sdhci_host *host = mmc_priv(mmc);
 943         int err;
 944
 945         err = sdhci_execute_tuning(mmc, opcode);
 946         if (!err && !host->tuning_err)
 947                 tegra_sdhci_post_tuning(host);
 948
 949         return err;
 950 }
 951
 952 static void tegra_sdhci_set_uhs_signaling(struct sdhci_host *host,
 953                                           unsigned timing)
 954 {
 955         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 956         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
 957         bool set_default_tap = false;
 958         bool set_dqs_trim = false;
 959         bool do_hs400_dll_cal = false;
 960         u8 iter = TRIES_256;
 961         u32 val;
 962
 963         tegra_host->ddr_signaling = false;
 964         switch (timing) {
 965         case MMC_TIMING_UHS_SDR50:
 966                 break;
 967         case MMC_TIMING_UHS_SDR104:
 968         case MMC_TIMING_MMC_HS200:
 969                 /* Don't set default tap on tunable modes. */
 970                 iter = TRIES_128;
 971                 break;
 972         case MMC_TIMING_MMC_HS400:
 973                 set_dqs_trim = true;
 974                 do_hs400_dll_cal = true;
 975                 iter = TRIES_128;
 976                 break;
 977         case MMC_TIMING_MMC_DDR52:
 978         case MMC_TIMING_UHS_DDR50:
 979                 tegra_host->ddr_signaling = true;
 980                 set_default_tap = true;
 981                 break;
 982         default:
 983                 set_default_tap = true;
 984                 break;
 985         }
 986
 987         val = sdhci_readl(host, SDHCI_VNDR_TUN_CTRL0_0);
 988         val &= ~(SDHCI_VNDR_TUN_CTRL0_TUN_ITER_MASK |
 989                  SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_MASK |
 990                  SDHCI_VNDR_TUN_CTRL0_MUL_M_MASK);
 991         val |= (iter << SDHCI_VNDR_TUN_CTRL0_TUN_ITER_SHIFT |
 992                 0 << SDHCI_VNDR_TUN_CTRL0_START_TAP_VAL_SHIFT |
 993                 1 << SDHCI_VNDR_TUN_CTRL0_MUL_M_SHIFT);
 994         sdhci_writel(host, val, SDHCI_VNDR_TUN_CTRL0_0);
 995         sdhci_writel(host, 0, SDHCI_TEGRA_VNDR_TUN_CTRL1_0);
 996
 997         host->tuning_loop_count = (iter == TRIES_128) ? 128 : 256;
 998
 999         sdhci_set_uhs_signaling(host, timing);
1000
1001         tegra_sdhci_pad_autocalib(host);
1002
1003         if (tegra_host->tuned_tap_delay && !set_default_tap)
1004                 tegra_sdhci_set_tap(host, tegra_host->tuned_tap_delay);
1005         else
1006                 tegra_sdhci_set_tap(host, tegra_host->default_tap);
1007
1008         if (set_dqs_trim)
1009                 tegra_sdhci_set_dqs_trim(host, tegra_host->dqs_trim);
1010
1011         if (do_hs400_dll_cal)
1012                 tegra_sdhci_hs400_dll_cal(host);
1013 }
1014
1015 static int tegra_sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
1016 {
1017         unsigned int min, max;
1018
1019         /*
1020          * Start search for minimum tap value at 10, as smaller values are
1021          * may wrongly be reported as working but fail at higher speeds,
1022          * according to the TRM.
1023          */
1024         min = 10;
1025         while (min < 255) {
1026                 tegra_sdhci_set_tap(host, min);
1027                 if (!mmc_send_tuning(host->mmc, opcode, NULL))
1028                         break;
1029                 min++;
1030         }
1031
1032         /* Find the maximum tap value that still passes. */
1033         max = min + 1;
1034         while (max < 255) {
1035                 tegra_sdhci_set_tap(host, max);
1036                 if (mmc_send_tuning(host->mmc, opcode, NULL)) {
1037                         max--;
1038                         break;
1039                 }
1040                 max++;
1041         }
1042
1043         /* The TRM states the ideal tap value is at 75% in the passing range. */
1044         tegra_sdhci_set_tap(host, min + ((max - min) * 3 / 4));
1045
1046         return mmc_send_tuning(host->mmc, opcode, NULL);
1047 }
1048
1049 static int sdhci_tegra_start_signal_voltage_switch(struct mmc_host *mmc,
1050                                                    struct mmc_ios *ios)
1051 {
1052         struct sdhci_host *host = mmc_priv(mmc);
1053         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1054         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1055         int ret = 0;
1056
1057         if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
1058                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1059                 if (ret < 0)
1060                         return ret;
1061                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1062         } else if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180) {
1063                 ret = sdhci_start_signal_voltage_switch(mmc, ios);
1064                 if (ret < 0)
1065                         return ret;
1066                 ret = tegra_sdhci_set_padctrl(host, ios->signal_voltage, true);
1067         }
1068
1069         if (tegra_host->pad_calib_required)
1070                 tegra_sdhci_pad_autocalib(host);
1071
1072         return ret;
1073 }
1074
1075 static int tegra_sdhci_init_pinctrl_info(struct device *dev,
1076                                          struct sdhci_tegra *tegra_host)
1077 {
1078         tegra_host->pinctrl_sdmmc = devm_pinctrl_get(dev);
1079         if (IS_ERR(tegra_host->pinctrl_sdmmc)) {
1080                 dev_dbg(dev, "No pinctrl info, err: %ld\n",
1081                         PTR_ERR(tegra_host->pinctrl_sdmmc));
1082                 return -1;
1083         }
1084
1085         tegra_host->pinctrl_state_1v8_drv = pinctrl_lookup_state(
1086                                 tegra_host->pinctrl_sdmmc, "sdmmc-1v8-drv");
1087         if (IS_ERR(tegra_host->pinctrl_state_1v8_drv)) {
1088                 if (PTR_ERR(tegra_host->pinctrl_state_1v8_drv) == -ENODEV)
1089                         tegra_host->pinctrl_state_1v8_drv = NULL;
1090         }
1091
1092         tegra_host->pinctrl_state_3v3_drv = pinctrl_lookup_state(
1093                                 tegra_host->pinctrl_sdmmc, "sdmmc-3v3-drv");
1094         if (IS_ERR(tegra_host->pinctrl_state_3v3_drv)) {
1095                 if (PTR_ERR(tegra_host->pinctrl_state_3v3_drv) == -ENODEV)
1096                         tegra_host->pinctrl_state_3v3_drv = NULL;
1097         }
1098
1099         tegra_host->pinctrl_state_3v3 =
1100                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-3v3");
1101         if (IS_ERR(tegra_host->pinctrl_state_3v3)) {
1102                 dev_warn(dev, "Missing 3.3V pad state, err: %ld\n",
1103                          PTR_ERR(tegra_host->pinctrl_state_3v3));
1104                 return -1;
1105         }
1106
1107         tegra_host->pinctrl_state_1v8 =
1108                 pinctrl_lookup_state(tegra_host->pinctrl_sdmmc, "sdmmc-1v8");
1109         if (IS_ERR(tegra_host->pinctrl_state_1v8)) {
1110                 dev_warn(dev, "Missing 1.8V pad state, err: %ld\n",
1111                          PTR_ERR(tegra_host->pinctrl_state_1v8));
1112                 return -1;
1113         }
1114
1115         tegra_host->pad_control_available = true;
1116
1117         return 0;
1118 }
1119
1120 static void tegra_sdhci_voltage_switch(struct sdhci_host *host)
1121 {
1122         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1123         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1124         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1125
1126         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1127                 tegra_host->pad_calib_required = true;
1128 }
1129
1130 static void tegra_cqhci_writel(struct cqhci_host *cq_host, u32 val, int reg)
1131 {
1132         struct mmc_host *mmc = cq_host->mmc;
1133         u8 ctrl;
1134         ktime_t timeout;
1135         bool timed_out;
1136
1137         /*
1138          * During CQE resume/unhalt, CQHCI driver unhalts CQE prior to
1139          * cqhci_host_ops enable where SDHCI DMA and BLOCK_SIZE registers need
1140          * to be re-configured.
1141          * Tegra CQHCI/SDHCI prevents write access to block size register when
1142          * CQE is unhalted. So handling CQE resume sequence here to configure
1143          * SDHCI block registers prior to exiting CQE halt state.
1144          */
1145         if (reg == CQHCI_CTL && !(val & CQHCI_HALT) &&
1146             cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) {
1147                 sdhci_cqe_enable(mmc);
1148                 writel(val, cq_host->mmio + reg);
1149                 timeout = ktime_add_us(ktime_get(), 50);
1150                 while (1) {
1151                         timed_out = ktime_compare(ktime_get(), timeout) > 0;
1152                         ctrl = cqhci_readl(cq_host, CQHCI_CTL);
1153                         if (!(ctrl & CQHCI_HALT) || timed_out)
1154                                 break;
1155                 }
1156                 /*
1157                  * CQE usually resumes very quick, but incase if Tegra CQE
1158                  * doesn't resume retry unhalt.
1159                  */
1160                 if (timed_out)
1161                         writel(val, cq_host->mmio + reg);
1162         } else {
1163                 writel(val, cq_host->mmio + reg);
1164         }
1165 }
1166
1167 static void sdhci_tegra_update_dcmd_desc(struct mmc_host *mmc,
1168                                          struct mmc_request *mrq, u64 *data)
1169 {
1170         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(mmc_priv(mmc));
1171         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1172         const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data;
1173
1174         if (soc_data->nvquirks & NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING &&
1175             mrq->cmd->flags & MMC_RSP_R1B)
1176                 *data |= CQHCI_CMD_TIMING(1);
1177 }
1178
1179 static void sdhci_tegra_cqe_enable(struct mmc_host *mmc)
1180 {
1181         struct cqhci_host *cq_host = mmc->cqe_private;
1182         u32 val;
1183
1184         /*
1185          * Tegra CQHCI/SDMMC design prevents write access to sdhci block size
1186          * register when CQE is enabled and unhalted.
1187          * CQHCI driver enables CQE prior to activation, so disable CQE before
1188          * programming block size in sdhci controller and enable it back.
1189          */
1190         if (!cq_host->activated) {
1191                 val = cqhci_readl(cq_host, CQHCI_CFG);
1192                 if (val & CQHCI_ENABLE)
1193                         cqhci_writel(cq_host, (val & ~CQHCI_ENABLE),
1194                                      CQHCI_CFG);
1195                 sdhci_cqe_enable(mmc);
1196                 if (val & CQHCI_ENABLE)
1197                         cqhci_writel(cq_host, val, CQHCI_CFG);
1198         }
1199
1200         /*
1201          * CMD CRC errors are seen sometimes with some eMMC devices when status
1202          * command is sent during transfer of last data block which is the
1203          * default case as send status command block counter (CBC) is 1.
1204          * Recommended fix to set CBC to 0 allowing send status command only
1205          * when data lines are idle.
1206          */
1207         val = cqhci_readl(cq_host, CQHCI_SSC1);
1208         val &= ~CQHCI_SSC1_CBC_MASK;
1209         cqhci_writel(cq_host, val, CQHCI_SSC1);
1210 }
1211
1212 static void sdhci_tegra_dumpregs(struct mmc_host *mmc)
1213 {
1214         sdhci_dumpregs(mmc_priv(mmc));
1215 }
1216
1217 static u32 sdhci_tegra_cqhci_irq(struct sdhci_host *host, u32 intmask)
1218 {
1219         int cmd_error = 0;
1220         int data_error = 0;
1221
1222         if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
1223                 return intmask;
1224
1225         cqhci_irq(host->mmc, intmask, cmd_error, data_error);
1226
1227         return 0;
1228 }
1229
1230 static const struct cqhci_host_ops sdhci_tegra_cqhci_ops = {
1231         .write_l    = tegra_cqhci_writel,
1232         .enable = sdhci_tegra_cqe_enable,
1233         .disable = sdhci_cqe_disable,
1234         .dumpregs = sdhci_tegra_dumpregs,
1235         .update_dcmd_desc = sdhci_tegra_update_dcmd_desc,
1236 };
1237
1238 static int tegra_sdhci_set_dma_mask(struct sdhci_host *host)
1239 {
1240         struct sdhci_pltfm_host *platform = sdhci_priv(host);
1241         struct sdhci_tegra *tegra = sdhci_pltfm_priv(platform);
1242         const struct sdhci_tegra_soc_data *soc = tegra->soc_data;
1243         struct device *dev = mmc_dev(host->mmc);
1244
1245         if (soc->dma_mask)
1246                 return dma_set_mask_and_coherent(dev, soc->dma_mask);
1247
1248         return 0;
1249 }
1250
1251 static const struct sdhci_ops tegra_sdhci_ops = {
1252         .get_ro     = tegra_sdhci_get_ro,
1253         .read_w     = tegra_sdhci_readw,
1254         .write_l    = tegra_sdhci_writel,
1255         .set_clock  = tegra_sdhci_set_clock,
1256         .set_dma_mask = tegra_sdhci_set_dma_mask,
1257         .set_bus_width = sdhci_set_bus_width,
1258         .reset      = tegra_sdhci_reset,
1259         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1260         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1261         .voltage_switch = tegra_sdhci_voltage_switch,
1262         .get_max_clock = tegra_sdhci_get_max_clock,
1263 };
1264
1265 static const struct sdhci_pltfm_data sdhci_tegra20_pdata = {
1266         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1267                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1268                   SDHCI_QUIRK_NO_HISPD_BIT |
1269                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1270                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1271         .ops  = &tegra_sdhci_ops,
1272 };
1273
1274 static const struct sdhci_tegra_soc_data soc_data_tegra20 = {
1275         .pdata = &sdhci_tegra20_pdata,
1276         .dma_mask = DMA_BIT_MASK(32),
1277         .nvquirks = NVQUIRK_FORCE_SDHCI_SPEC_200 |
1278                     NVQUIRK_ENABLE_BLOCK_GAP_DET,
1279 };
1280
1281 static const struct sdhci_pltfm_data sdhci_tegra30_pdata = {
1282         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1283                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1284                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1285                   SDHCI_QUIRK_NO_HISPD_BIT |
1286                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1287                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1288         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
1289                    SDHCI_QUIRK2_BROKEN_HS200 |
1290                    /*
1291                     * Auto-CMD23 leads to "Got command interrupt 0x00010000 even
1292                     * though no command operation was in progress."
1293                     *
1294                     * The exact reason is unknown, as the same hardware seems
1295                     * to support Auto CMD23 on a downstream 3.1 kernel.
1296                     */
1297                    SDHCI_QUIRK2_ACMD23_BROKEN,
1298         .ops  = &tegra_sdhci_ops,
1299 };
1300
1301 static const struct sdhci_tegra_soc_data soc_data_tegra30 = {
1302         .pdata = &sdhci_tegra30_pdata,
1303         .dma_mask = DMA_BIT_MASK(32),
1304         .nvquirks = NVQUIRK_ENABLE_SDHCI_SPEC_300 |
1305                     NVQUIRK_ENABLE_SDR50 |
1306                     NVQUIRK_ENABLE_SDR104 |
1307                     NVQUIRK_HAS_PADCALIB,
1308 };
1309
1310 static const struct sdhci_ops tegra114_sdhci_ops = {
1311         .get_ro     = tegra_sdhci_get_ro,
1312         .read_w     = tegra_sdhci_readw,
1313         .write_w    = tegra_sdhci_writew,
1314         .write_l    = tegra_sdhci_writel,
1315         .set_clock  = tegra_sdhci_set_clock,
1316         .set_dma_mask = tegra_sdhci_set_dma_mask,
1317         .set_bus_width = sdhci_set_bus_width,
1318         .reset      = tegra_sdhci_reset,
1319         .platform_execute_tuning = tegra_sdhci_execute_tuning,
1320         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1321         .voltage_switch = tegra_sdhci_voltage_switch,
1322         .get_max_clock = tegra_sdhci_get_max_clock,
1323 };
1324
1325 static const struct sdhci_pltfm_data sdhci_tegra114_pdata = {
1326         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1327                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1328                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1329                   SDHCI_QUIRK_NO_HISPD_BIT |
1330                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1331                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1332         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1333         .ops  = &tegra114_sdhci_ops,
1334 };
1335
1336 static const struct sdhci_tegra_soc_data soc_data_tegra114 = {
1337         .pdata = &sdhci_tegra114_pdata,
1338         .dma_mask = DMA_BIT_MASK(32),
1339 };
1340
1341 static const struct sdhci_pltfm_data sdhci_tegra124_pdata = {
1342         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1343                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1344                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1345                   SDHCI_QUIRK_NO_HISPD_BIT |
1346                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1347                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1348         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1349         .ops  = &tegra114_sdhci_ops,
1350 };
1351
1352 static const struct sdhci_tegra_soc_data soc_data_tegra124 = {
1353         .pdata = &sdhci_tegra124_pdata,
1354         .dma_mask = DMA_BIT_MASK(34),
1355 };
1356
1357 static const struct sdhci_ops tegra210_sdhci_ops = {
1358         .get_ro     = tegra_sdhci_get_ro,
1359         .read_w     = tegra_sdhci_readw,
1360         .write_w    = tegra210_sdhci_writew,
1361         .write_l    = tegra_sdhci_writel,
1362         .set_clock  = tegra_sdhci_set_clock,
1363         .set_dma_mask = tegra_sdhci_set_dma_mask,
1364         .set_bus_width = sdhci_set_bus_width,
1365         .reset      = tegra_sdhci_reset,
1366         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1367         .voltage_switch = tegra_sdhci_voltage_switch,
1368         .get_max_clock = tegra_sdhci_get_max_clock,
1369 };
1370
1371 static const struct sdhci_pltfm_data sdhci_tegra210_pdata = {
1372         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1373                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1374                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1375                   SDHCI_QUIRK_NO_HISPD_BIT |
1376                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1377                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1378         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1379         .ops  = &tegra210_sdhci_ops,
1380 };
1381
1382 static const struct sdhci_tegra_soc_data soc_data_tegra210 = {
1383         .pdata = &sdhci_tegra210_pdata,
1384         .dma_mask = DMA_BIT_MASK(34),
1385         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1386                     NVQUIRK_HAS_PADCALIB |
1387                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1388                     NVQUIRK_ENABLE_SDR50 |
1389                     NVQUIRK_ENABLE_SDR104,
1390         .min_tap_delay = 106,
1391         .max_tap_delay = 185,
1392 };
1393
1394 static const struct sdhci_ops tegra186_sdhci_ops = {
1395         .get_ro     = tegra_sdhci_get_ro,
1396         .read_w     = tegra_sdhci_readw,
1397         .write_l    = tegra_sdhci_writel,
1398         .set_clock  = tegra_sdhci_set_clock,
1399         .set_dma_mask = tegra_sdhci_set_dma_mask,
1400         .set_bus_width = sdhci_set_bus_width,
1401         .reset      = tegra_sdhci_reset,
1402         .set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
1403         .voltage_switch = tegra_sdhci_voltage_switch,
1404         .get_max_clock = tegra_sdhci_get_max_clock,
1405         .irq = sdhci_tegra_cqhci_irq,
1406 };
1407
1408 static const struct sdhci_pltfm_data sdhci_tegra186_pdata = {
1409         .quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
1410                   SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
1411                   SDHCI_QUIRK_SINGLE_POWER_WRITE |
1412                   SDHCI_QUIRK_NO_HISPD_BIT |
1413                   SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
1414                   SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
1415         .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
1416         .ops  = &tegra186_sdhci_ops,
1417 };
1418
1419 static const struct sdhci_tegra_soc_data soc_data_tegra186 = {
1420         .pdata = &sdhci_tegra186_pdata,
1421         .dma_mask = DMA_BIT_MASK(40),
1422         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1423                     NVQUIRK_HAS_PADCALIB |
1424                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1425                     NVQUIRK_ENABLE_SDR50 |
1426                     NVQUIRK_ENABLE_SDR104 |
1427                     NVQUIRK_CQHCI_DCMD_R1B_CMD_TIMING,
1428         .min_tap_delay = 84,
1429         .max_tap_delay = 136,
1430 };
1431
1432 static const struct sdhci_tegra_soc_data soc_data_tegra194 = {
1433         .pdata = &sdhci_tegra186_pdata,
1434         .dma_mask = DMA_BIT_MASK(39),
1435         .nvquirks = NVQUIRK_NEEDS_PAD_CONTROL |
1436                     NVQUIRK_HAS_PADCALIB |
1437                     NVQUIRK_DIS_CARD_CLK_CONFIG_TAP |
1438                     NVQUIRK_ENABLE_SDR50 |
1439                     NVQUIRK_ENABLE_SDR104,
1440         .min_tap_delay = 96,
1441         .max_tap_delay = 139,
1442 };
1443
1444 static const struct of_device_id sdhci_tegra_dt_match[] = {
1445         { .compatible = "nvidia,tegra194-sdhci", .data = &soc_data_tegra194 },
1446         { .compatible = "nvidia,tegra186-sdhci", .data = &soc_data_tegra186 },
1447         { .compatible = "nvidia,tegra210-sdhci", .data = &soc_data_tegra210 },
1448         { .compatible = "nvidia,tegra124-sdhci", .data = &soc_data_tegra124 },
1449         { .compatible = "nvidia,tegra114-sdhci", .data = &soc_data_tegra114 },
1450         { .compatible = "nvidia,tegra30-sdhci", .data = &soc_data_tegra30 },
1451         { .compatible = "nvidia,tegra20-sdhci", .data = &soc_data_tegra20 },
1452         {}
1453 };
1454 MODULE_DEVICE_TABLE(of, sdhci_tegra_dt_match);
1455
1456 static int sdhci_tegra_add_host(struct sdhci_host *host)
1457 {
1458         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1459         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1460         struct cqhci_host *cq_host;
1461         bool dma64;
1462         int ret;
1463
1464         if (!tegra_host->enable_hwcq)
1465                 return sdhci_add_host(host);
1466
1467         sdhci_enable_v4_mode(host);
1468
1469         ret = sdhci_setup_host(host);
1470         if (ret)
1471                 return ret;
1472
1473         host->mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
1474
1475         cq_host = devm_kzalloc(host->mmc->parent,
1476                                 sizeof(*cq_host), GFP_KERNEL);
1477         if (!cq_host) {
1478                 ret = -ENOMEM;
1479                 goto cleanup;
1480         }
1481
1482         cq_host->mmio = host->ioaddr + SDHCI_TEGRA_CQE_BASE_ADDR;
1483         cq_host->ops = &sdhci_tegra_cqhci_ops;
1484
1485         dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
1486         if (dma64)
1487                 cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
1488
1489         ret = cqhci_init(cq_host, host->mmc, dma64);
1490         if (ret)
1491                 goto cleanup;
1492
1493         ret = __sdhci_add_host(host);
1494         if (ret)
1495                 goto cleanup;
1496
1497         return 0;
1498
1499 cleanup:
1500         sdhci_cleanup_host(host);
1501         return ret;
1502 }
1503
1504 static int sdhci_tegra_probe(struct platform_device *pdev)
1505 {
1506         const struct of_device_id *match;
1507         const struct sdhci_tegra_soc_data *soc_data;
1508         struct sdhci_host *host;
1509         struct sdhci_pltfm_host *pltfm_host;
1510         struct sdhci_tegra *tegra_host;
1511         struct clk *clk;
1512         int rc;
1513
1514         match = of_match_device(sdhci_tegra_dt_match, &pdev->dev);
1515         if (!match)
1516                 return -EINVAL;
1517         soc_data = match->data;
1518
1519         host = sdhci_pltfm_init(pdev, soc_data->pdata, sizeof(*tegra_host));
1520         if (IS_ERR(host))
1521                 return PTR_ERR(host);
1522         pltfm_host = sdhci_priv(host);
1523
1524         tegra_host = sdhci_pltfm_priv(pltfm_host);
1525         tegra_host->ddr_signaling = false;
1526         tegra_host->pad_calib_required = false;
1527         tegra_host->pad_control_available = false;
1528         tegra_host->soc_data = soc_data;
1529
1530         if (soc_data->nvquirks & NVQUIRK_NEEDS_PAD_CONTROL) {
1531                 rc = tegra_sdhci_init_pinctrl_info(&pdev->dev, tegra_host);
1532                 if (rc == 0)
1533                         host->mmc_host_ops.start_signal_voltage_switch =
1534                                 sdhci_tegra_start_signal_voltage_switch;
1535         }
1536
1537         /* Hook to periodically rerun pad calibration */
1538         if (soc_data->nvquirks & NVQUIRK_HAS_PADCALIB)
1539                 host->mmc_host_ops.request = tegra_sdhci_request;
1540
1541         host->mmc_host_ops.hs400_enhanced_strobe =
1542                         tegra_sdhci_hs400_enhanced_strobe;
1543
1544         if (!host->ops->platform_execute_tuning)
1545                 host->mmc_host_ops.execute_tuning =
1546                                 tegra_sdhci_execute_hw_tuning;
1547
1548         rc = mmc_of_parse(host->mmc);
1549         if (rc)
1550                 goto err_parse_dt;
1551
1552         if (tegra_host->soc_data->nvquirks & NVQUIRK_ENABLE_DDR50)
1553                 host->mmc->caps |= MMC_CAP_1_8V_DDR;
1554
1555         tegra_sdhci_parse_dt(host);
1556
1557         tegra_host->power_gpio = devm_gpiod_get_optional(&pdev->dev, "power",
1558                                                          GPIOD_OUT_HIGH);
1559         if (IS_ERR(tegra_host->power_gpio)) {
1560                 rc = PTR_ERR(tegra_host->power_gpio);
1561                 goto err_power_req;
1562         }
1563
1564         clk = devm_clk_get(mmc_dev(host->mmc), NULL);
1565         if (IS_ERR(clk)) {
1566                 rc = PTR_ERR(clk);
1567
1568                 if (rc != -EPROBE_DEFER)
1569                         dev_err(&pdev->dev, "failed to get clock: %d\n", rc);
1570
1571                 goto err_clk_get;
1572         }
1573         clk_prepare_enable(clk);
1574         pltfm_host->clk = clk;
1575
1576         tegra_host->rst = devm_reset_control_get_exclusive(&pdev->dev,
1577                                                            "sdhci");
1578         if (IS_ERR(tegra_host->rst)) {
1579                 rc = PTR_ERR(tegra_host->rst);
1580                 dev_err(&pdev->dev, "failed to get reset control: %d\n", rc);
1581                 goto err_rst_get;
1582         }
1583
1584         rc = reset_control_assert(tegra_host->rst);
1585         if (rc)
1586                 goto err_rst_get;
1587
1588         usleep_range(2000, 4000);
1589
1590         rc = reset_control_deassert(tegra_host->rst);
1591         if (rc)
1592                 goto err_rst_get;
1593
1594         usleep_range(2000, 4000);
1595
1596         rc = sdhci_tegra_add_host(host);
1597         if (rc)
1598                 goto err_add_host;
1599
1600         return 0;
1601
1602 err_add_host:
1603         reset_control_assert(tegra_host->rst);
1604 err_rst_get:
1605         clk_disable_unprepare(pltfm_host->clk);
1606 err_clk_get:
1607 err_power_req:
1608 err_parse_dt:
1609         sdhci_pltfm_free(pdev);
1610         return rc;
1611 }
1612
1613 static int sdhci_tegra_remove(struct platform_device *pdev)
1614 {
1615         struct sdhci_host *host = platform_get_drvdata(pdev);
1616         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1617         struct sdhci_tegra *tegra_host = sdhci_pltfm_priv(pltfm_host);
1618
1619         sdhci_remove_host(host, 0);
1620
1621         reset_control_assert(tegra_host->rst);
1622         usleep_range(2000, 4000);
1623         clk_disable_unprepare(pltfm_host->clk);
1624
1625         sdhci_pltfm_free(pdev);
1626
1627         return 0;
1628 }
1629
1630 #ifdef CONFIG_PM_SLEEP
1631 static int __maybe_unused sdhci_tegra_suspend(struct device *dev)
1632 {
1633         struct sdhci_host *host = dev_get_drvdata(dev);
1634         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1635         int ret;
1636
1637         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1638                 ret = cqhci_suspend(host->mmc);
1639                 if (ret)
1640                         return ret;
1641         }
1642
1643         ret = sdhci_suspend_host(host);
1644         if (ret) {
1645                 cqhci_resume(host->mmc);
1646                 return ret;
1647         }
1648
1649         clk_disable_unprepare(pltfm_host->clk);
1650         return 0;
1651 }
1652
1653 static int __maybe_unused sdhci_tegra_resume(struct device *dev)
1654 {
1655         struct sdhci_host *host = dev_get_drvdata(dev);
1656         struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
1657         int ret;
1658
1659         ret = clk_prepare_enable(pltfm_host->clk);
1660         if (ret)
1661                 return ret;
1662
1663         ret = sdhci_resume_host(host);
1664         if (ret)
1665                 goto disable_clk;
1666
1667         if (host->mmc->caps2 & MMC_CAP2_CQE) {
1668                 ret = cqhci_resume(host->mmc);
1669                 if (ret)
1670                         goto suspend_host;
1671         }
1672
1673         return 0;
1674
1675 suspend_host:
1676         sdhci_suspend_host(host);
1677 disable_clk:
1678         clk_disable_unprepare(pltfm_host->clk);
1679         return ret;
1680 }
1681 #endif
1682
1683 static SIMPLE_DEV_PM_OPS(sdhci_tegra_dev_pm_ops, sdhci_tegra_suspend,
1684                          sdhci_tegra_resume);
1685
1686 static struct platform_driver sdhci_tegra_driver = {
1687         .driver         = {
1688                 .name   = "sdhci-tegra",
1689                 .of_match_table = sdhci_tegra_dt_match,
1690                 .pm     = &sdhci_tegra_dev_pm_ops,
1691         },
1692         .probe          = sdhci_tegra_probe,
1693         .remove         = sdhci_tegra_remove,
1694 };
1695
1696 module_platform_driver(sdhci_tegra_driver);
1697
1698 MODULE_DESCRIPTION("SDHCI driver for Tegra");
1699 MODULE_AUTHOR("Google, Inc.");
1700 MODULE_LICENSE("GPL v2");