drivers/scsi/ufs/ufshcd.c

   1 /*
   2  * Universal Flash Storage Host controller driver Core
   3  *
   4  * This code is based on drivers/scsi/ufs/ufshcd.c
   5  * Copyright (C) 2011-2013 Samsung India Software Operations
   6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   7  *
   8  * Authors:
   9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
  10  *      Vinayak Holikatti <h.vinayak@samsung.com>
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  * See the COPYING file in the top-level directory or visit
  17  * <http://www.gnu.org/licenses/gpl-2.0.html>
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
  25  * without warranty of any kind. You are solely responsible for
  26  * determining the appropriateness of using and distributing
  27  * the program and assume all risks associated with your exercise
  28  * of rights with respect to the program, including but not limited
  29  * to infringement of third party rights, the risks and costs of
  30  * program errors, damage to or loss of data, programs or equipment,
  31  * and unavailability or interruption of operations. Under no
  32  * circumstances will the contributor of this Program be liable for
  33  * any damages of any kind arising from your use or distribution of
  34  * this program.
  35  *
  36  * The Linux Foundation chooses to take subject only to the GPLv2
  37  * license terms, and distributes only under these terms.
  38  */
  39
  40 #include <linux/async.h>
  41 #include <linux/devfreq.h>
  42 #include <linux/nls.h>
  43 #include <linux/of.h>
  44 #include <linux/bitfield.h>
  45 #include "ufshcd.h"
  46 #include "ufs_quirks.h"
  47 #include "unipro.h"
  48 #include "ufs-sysfs.h"
  49 #include "ufs_bsg.h"
  50
  51 #define CREATE_TRACE_POINTS
  52 #include <trace/events/ufs.h>
  53
  54 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  55                                  UTP_TASK_REQ_COMPL |\
  56                                  UFSHCD_ERROR_MASK)
  57 /* UIC command timeout, unit: ms */
  58 #define UIC_CMD_TIMEOUT 500
  59
  60 /* NOP OUT retries waiting for NOP IN response */
  61 #define NOP_OUT_RETRIES    10
  62 /* Timeout after 30 msecs if NOP OUT hangs without response */
  63 #define NOP_OUT_TIMEOUT    30 /* msecs */
  64
  65 /* Query request retries */
  66 #define QUERY_REQ_RETRIES 3
  67 /* Query request timeout */
  68 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  69
  70 /* Task management command timeout */
  71 #define TM_CMD_TIMEOUT  100 /* msecs */
  72
  73 /* maximum number of retries for a general UIC command  */
  74 #define UFS_UIC_COMMAND_RETRIES 3
  75
  76 /* maximum number of link-startup retries */
  77 #define DME_LINKSTARTUP_RETRIES 3
  78
  79 /* Maximum retries for Hibern8 enter */
  80 #define UIC_HIBERN8_ENTER_RETRIES 3
  81
  82 /* maximum number of reset retries before giving up */
  83 #define MAX_HOST_RESET_RETRIES 5
  84
  85 /* Expose the flag value from utp_upiu_query.value */
  86 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  87
  88 /* Interrupt aggregation default timeout, unit: 40us */
  89 #define INT_AGGR_DEF_TO 0x02
  90
  91 /* default delay of autosuspend: 2000 ms */
  92 #define RPM_AUTOSUSPEND_DELAY_MS 2000
  93
  94 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  95         ({                                                              \
  96                 int _ret;                                               \
  97                 if (_on)                                                \
  98                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  99                 else                                                    \
 100                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
 101                 _ret;                                                   \
 102         })
 103
 104 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
 105         size_t __len = (len);                                            \
 106         print_hex_dump(KERN_ERR, prefix_str,                             \
 107                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
 108                        16, 4, buf, __len, false);                        \
 109 } while (0)
 110
 111 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 112                      const char *prefix)
 113 {
 114         u32 *regs;
 115         size_t pos;
 116
 117         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 118                 return -EINVAL;
 119
 120         regs = kzalloc(len, GFP_ATOMIC);
 121         if (!regs)
 122                 return -ENOMEM;
 123
 124         for (pos = 0; pos < len; pos += 4)
 125                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 126
 127         ufshcd_hex_dump(prefix, regs, len);
 128         kfree(regs);
 129
 130         return 0;
 131 }
 132 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 133
 134 enum {
 135         UFSHCD_MAX_CHANNEL      = 0,
 136         UFSHCD_MAX_ID           = 1,
 137         UFSHCD_CMD_PER_LUN      = 32,
 138         UFSHCD_CAN_QUEUE        = 32,
 139 };
 140
 141 /* UFSHCD states */
 142 enum {
 143         UFSHCD_STATE_RESET,
 144         UFSHCD_STATE_ERROR,
 145         UFSHCD_STATE_OPERATIONAL,
 146         UFSHCD_STATE_EH_SCHEDULED,
 147 };
 148
 149 /* UFSHCD error handling flags */
 150 enum {
 151         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 152 };
 153
 154 /* UFSHCD UIC layer error flags */
 155 enum {
 156         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 157         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 158         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 159         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 160         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 161         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 162 };
 163
 164 #define ufshcd_set_eh_in_progress(h) \
 165         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 166 #define ufshcd_eh_in_progress(h) \
 167         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 168 #define ufshcd_clear_eh_in_progress(h) \
 169         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 170
 171 #define ufshcd_set_ufs_dev_active(h) \
 172         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
 173 #define ufshcd_set_ufs_dev_sleep(h) \
 174         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
 175 #define ufshcd_set_ufs_dev_poweroff(h) \
 176         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
 177 #define ufshcd_is_ufs_dev_active(h) \
 178         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
 179 #define ufshcd_is_ufs_dev_sleep(h) \
 180         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
 181 #define ufshcd_is_ufs_dev_poweroff(h) \
 182         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
 183
 184 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 185         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 186         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 187         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 188         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 189         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 190         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 191 };
 192
 193 static inline enum ufs_dev_pwr_mode
 194 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 195 {
 196         return ufs_pm_lvl_states[lvl].dev_state;
 197 }
 198
 199 static inline enum uic_link_state
 200 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 201 {
 202         return ufs_pm_lvl_states[lvl].link_state;
 203 }
 204
 205 static inline enum ufs_pm_level
 206 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 207                                         enum uic_link_state link_state)
 208 {
 209         enum ufs_pm_level lvl;
 210
 211         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 212                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 213                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 214                         return lvl;
 215         }
 216
 217         /* if no match found, return the level 0 */
 218         return UFS_PM_LVL_0;
 219 }
 220
 221 static struct ufs_dev_fix ufs_fixups[] = {
 222         /* UFS cards deviations table */
 223         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 224                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 225         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 226                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
 227         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 228                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
 229         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
 230                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 231         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
 232                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 233         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
 234                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 235         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
 236                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
 237         UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
 238                 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
 239
 240         END_FIX
 241 };
 242
 243 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 244 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 245 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 246 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 247 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 248 static void ufshcd_hba_exit(struct ufs_hba *hba);
 249 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
 250 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 251                                  bool skip_ref_clk);
 252 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 253 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
 254 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 255 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 256 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 257 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 258 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 259 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 260 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 261 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 262                              struct ufs_pa_layer_attr *pwr_mode);
 263 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
 264 {
 265         return tag >= 0 && tag < hba->nutrs;
 266 }
 267
 268 static inline void ufshcd_enable_irq(struct ufs_hba *hba)
 269 {
 270         if (!hba->is_irq_enabled) {
 271                 enable_irq(hba->irq);
 272                 hba->is_irq_enabled = true;
 273         }
 274 }
 275
 276 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 277 {
 278         if (hba->is_irq_enabled) {
 279                 disable_irq(hba->irq);
 280                 hba->is_irq_enabled = false;
 281         }
 282 }
 283
 284 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 285 {
 286         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 287                 scsi_unblock_requests(hba->host);
 288 }
 289
 290 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 291 {
 292         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 293                 scsi_block_requests(hba->host);
 294 }
 295
 296 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 297                 const char *str)
 298 {
 299         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 300
 301         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
 302 }
 303
 304 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 305                 const char *str)
 306 {
 307         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 308
 309         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
 310 }
 311
 312 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 313                 const char *str)
 314 {
 315         int off = (int)tag - hba->nutrs;
 316         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
 317
 318         trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
 319                         &descp->input_param1);
 320 }
 321
 322 static void ufshcd_add_command_trace(struct ufs_hba *hba,
 323                 unsigned int tag, const char *str)
 324 {
 325         sector_t lba = -1;
 326         u8 opcode = 0;
 327         u32 intr, doorbell;
 328         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 329         struct scsi_cmnd *cmd = lrbp->cmd;
 330         int transfer_len = -1;
 331
 332         if (!trace_ufshcd_command_enabled()) {
 333                 /* trace UPIU W/O tracing command */
 334                 if (cmd)
 335                         ufshcd_add_cmd_upiu_trace(hba, tag, str);
 336                 return;
 337         }
 338
 339         if (cmd) { /* data phase exists */
 340                 /* trace UPIU also */
 341                 ufshcd_add_cmd_upiu_trace(hba, tag, str);
 342                 opcode = cmd->cmnd[0];
 343                 if ((opcode == READ_10) || (opcode == WRITE_10)) {
 344                         /*
 345                          * Currently we only fully trace read(10) and write(10)
 346                          * commands
 347                          */
 348                         if (cmd->request && cmd->request->bio)
 349                                 lba = cmd->request->bio->bi_iter.bi_sector;
 350                         transfer_len = be32_to_cpu(
 351                                 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 352                 }
 353         }
 354
 355         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 356         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 357         trace_ufshcd_command(dev_name(hba->dev), str, tag,
 358                                 doorbell, transfer_len, intr, lba, opcode);
 359 }
 360
 361 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 362 {
 363         struct ufs_clk_info *clki;
 364         struct list_head *head = &hba->clk_list_head;
 365
 366         if (list_empty(head))
 367                 return;
 368
 369         list_for_each_entry(clki, head, list) {
 370                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 371                                 clki->max_freq)
 372                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 373                                         clki->name, clki->curr_freq);
 374         }
 375 }
 376
 377 static void ufshcd_print_err_hist(struct ufs_hba *hba,
 378                                   struct ufs_err_reg_hist *err_hist,
 379                                   char *err_name)
 380 {
 381         int i;
 382         bool found = false;
 383
 384         for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
 385                 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
 386
 387                 if (err_hist->tstamp[p] == 0)
 388                         continue;
 389                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 390                         err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
 391                 found = true;
 392         }
 393
 394         if (!found)
 395                 dev_err(hba->dev, "No record of %s\n", err_name);
 396 }
 397
 398 static void ufshcd_print_host_regs(struct ufs_hba *hba)
 399 {
 400         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 401         dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
 402                 hba->ufs_version, hba->capabilities);
 403         dev_err(hba->dev,
 404                 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
 405                 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
 406         dev_err(hba->dev,
 407                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
 408                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
 409                 hba->ufs_stats.hibern8_exit_cnt);
 410
 411         ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
 412         ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
 413         ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
 414         ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
 415         ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
 416         ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
 417                               "auto_hibern8_err");
 418         ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
 419         ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
 420                               "link_startup_fail");
 421         ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
 422         ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
 423                               "suspend_fail");
 424         ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
 425         ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
 426         ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
 427
 428         ufshcd_print_clk_freqs(hba);
 429
 430         ufshcd_vops_dbg_register_dump(hba);
 431 }
 432
 433 static
 434 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
 435 {
 436         struct ufshcd_lrb *lrbp;
 437         int prdt_length;
 438         int tag;
 439
 440         for_each_set_bit(tag, &bitmap, hba->nutrs) {
 441                 lrbp = &hba->lrb[tag];
 442
 443                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 444                                 tag, ktime_to_us(lrbp->issue_time_stamp));
 445                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 446                                 tag, ktime_to_us(lrbp->compl_time_stamp));
 447                 dev_err(hba->dev,
 448                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 449                         tag, (u64)lrbp->utrd_dma_addr);
 450
 451                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 452                                 sizeof(struct utp_transfer_req_desc));
 453                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 454                         (u64)lrbp->ucd_req_dma_addr);
 455                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 456                                 sizeof(struct utp_upiu_req));
 457                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 458                         (u64)lrbp->ucd_rsp_dma_addr);
 459                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 460                                 sizeof(struct utp_upiu_rsp));
 461
 462                 prdt_length = le16_to_cpu(
 463                         lrbp->utr_descriptor_ptr->prd_table_length);
 464                 dev_err(hba->dev,
 465                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 466                         tag, prdt_length,
 467                         (u64)lrbp->ucd_prdt_dma_addr);
 468
 469                 if (pr_prdt)
 470                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 471                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
 472         }
 473 }
 474
 475 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 476 {
 477         int tag;
 478
 479         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 480                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 481
 482                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 483                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 484         }
 485 }
 486
 487 static void ufshcd_print_host_state(struct ufs_hba *hba)
 488 {
 489         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 490         dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 491                 hba->outstanding_reqs, hba->outstanding_tasks);
 492         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 493                 hba->saved_err, hba->saved_uic_err);
 494         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 495                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 496         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 497                 hba->pm_op_in_progress, hba->is_sys_suspended);
 498         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 499                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 500         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 501         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 502                 hba->eh_flags, hba->req_abort_count);
 503         dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
 504                 hba->capabilities, hba->caps);
 505         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 506                 hba->dev_quirks);
 507 }
 508
 509 /**
 510  * ufshcd_print_pwr_info - print power params as saved in hba
 511  * power info
 512  * @hba: per-adapter instance
 513  */
 514 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 515 {
 516         static const char * const names[] = {
 517                 "INVALID MODE",
 518                 "FAST MODE",
 519                 "SLOW_MODE",
 520                 "INVALID MODE",
 521                 "FASTAUTO_MODE",
 522                 "SLOWAUTO_MODE",
 523                 "INVALID MODE",
 524         };
 525
 526         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 527                  __func__,
 528                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 529                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 530                  names[hba->pwr_info.pwr_rx],
 531                  names[hba->pwr_info.pwr_tx],
 532                  hba->pwr_info.hs_rate);
 533 }
 534
 535 /*
 536  * ufshcd_wait_for_register - wait for register value to change
 537  * @hba - per-adapter interface
 538  * @reg - mmio register offset
 539  * @mask - mask to apply to read register value
 540  * @val - wait condition
 541  * @interval_us - polling interval in microsecs
 542  * @timeout_ms - timeout in millisecs
 543  * @can_sleep - perform sleep or just spin
 544  *
 545  * Returns -ETIMEDOUT on error, zero on success
 546  */
 547 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 548                                 u32 val, unsigned long interval_us,
 549                                 unsigned long timeout_ms, bool can_sleep)
 550 {
 551         int err = 0;
 552         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 553
 554         /* ignore bits that we don't intend to wait on */
 555         val = val & mask;
 556
 557         while ((ufshcd_readl(hba, reg) & mask) != val) {
 558                 if (can_sleep)
 559                         usleep_range(interval_us, interval_us + 50);
 560                 else
 561                         udelay(interval_us);
 562                 if (time_after(jiffies, timeout)) {
 563                         if ((ufshcd_readl(hba, reg) & mask) != val)
 564                                 err = -ETIMEDOUT;
 565                         break;
 566                 }
 567         }
 568
 569         return err;
 570 }
 571
 572 /**
 573  * ufshcd_get_intr_mask - Get the interrupt bit mask
 574  * @hba: Pointer to adapter instance
 575  *
 576  * Returns interrupt bit mask per version
 577  */
 578 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 579 {
 580         u32 intr_mask = 0;
 581
 582         switch (hba->ufs_version) {
 583         case UFSHCI_VERSION_10:
 584                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
 585                 break;
 586         case UFSHCI_VERSION_11:
 587         case UFSHCI_VERSION_20:
 588                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
 589                 break;
 590         case UFSHCI_VERSION_21:
 591         default:
 592                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
 593                 break;
 594         }
 595
 596         return intr_mask;
 597 }
 598
 599 /**
 600  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 601  * @hba: Pointer to adapter instance
 602  *
 603  * Returns UFSHCI version supported by the controller
 604  */
 605 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 606 {
 607         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 608                 return ufshcd_vops_get_ufs_hci_version(hba);
 609
 610         return ufshcd_readl(hba, REG_UFS_VERSION);
 611 }
 612
 613 /**
 614  * ufshcd_is_device_present - Check if any device connected to
 615  *                            the host controller
 616  * @hba: pointer to adapter instance
 617  *
 618  * Returns true if device present, false if no device detected
 619  */
 620 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 621 {
 622         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 623                                                 DEVICE_PRESENT) ? true : false;
 624 }
 625
 626 /**
 627  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 628  * @lrbp: pointer to local command reference block
 629  *
 630  * This function is used to get the OCS field from UTRD
 631  * Returns the OCS field in the UTRD
 632  */
 633 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 634 {
 635         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 636 }
 637
 638 /**
 639  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 640  * @hba: per adapter instance
 641  * @pos: position of the bit to be cleared
 642  */
 643 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 644 {
 645         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 646                 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 647         else
 648                 ufshcd_writel(hba, ~(1 << pos),
 649                                 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 650 }
 651
 652 /**
 653  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
 654  * @hba: per adapter instance
 655  * @pos: position of the bit to be cleared
 656  */
 657 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 658 {
 659         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 660                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 661         else
 662                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 663 }
 664
 665 /**
 666  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
 667  * @hba: per adapter instance
 668  * @tag: position of the bit to be cleared
 669  */
 670 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
 671 {
 672         __clear_bit(tag, &hba->outstanding_reqs);
 673 }
 674
 675 /**
 676  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 677  * @reg: Register value of host controller status
 678  *
 679  * Returns integer, 0 on Success and positive value if failed
 680  */
 681 static inline int ufshcd_get_lists_status(u32 reg)
 682 {
 683         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 684 }
 685
 686 /**
 687  * ufshcd_get_uic_cmd_result - Get the UIC command result
 688  * @hba: Pointer to adapter instance
 689  *
 690  * This function gets the result of UIC command completion
 691  * Returns 0 on success, non zero value on error
 692  */
 693 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 694 {
 695         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 696                MASK_UIC_COMMAND_RESULT;
 697 }
 698
 699 /**
 700  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 701  * @hba: Pointer to adapter instance
 702  *
 703  * This function gets UIC command argument3
 704  * Returns 0 on success, non zero value on error
 705  */
 706 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 707 {
 708         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 709 }
 710
 711 /**
 712  * ufshcd_get_req_rsp - returns the TR response transaction type
 713  * @ucd_rsp_ptr: pointer to response UPIU
 714  */
 715 static inline int
 716 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 717 {
 718         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 719 }
 720
 721 /**
 722  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 723  * @ucd_rsp_ptr: pointer to response UPIU
 724  *
 725  * This function gets the response status and scsi_status from response UPIU
 726  * Returns the response result code.
 727  */
 728 static inline int
 729 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 730 {
 731         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 732 }
 733
 734 /*
 735  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 736  *                              from response UPIU
 737  * @ucd_rsp_ptr: pointer to response UPIU
 738  *
 739  * Return the data segment length.
 740  */
 741 static inline unsigned int
 742 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 743 {
 744         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 745                 MASK_RSP_UPIU_DATA_SEG_LEN;
 746 }
 747
 748 /**
 749  * ufshcd_is_exception_event - Check if the device raised an exception event
 750  * @ucd_rsp_ptr: pointer to response UPIU
 751  *
 752  * The function checks if the device raised an exception event indicated in
 753  * the Device Information field of response UPIU.
 754  *
 755  * Returns true if exception is raised, false otherwise.
 756  */
 757 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 758 {
 759         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 760                         MASK_RSP_EXCEPTION_EVENT ? true : false;
 761 }
 762
 763 /**
 764  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 765  * @hba: per adapter instance
 766  */
 767 static inline void
 768 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 769 {
 770         ufshcd_writel(hba, INT_AGGR_ENABLE |
 771                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 772                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 773 }
 774
 775 /**
 776  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 777  * @hba: per adapter instance
 778  * @cnt: Interrupt aggregation counter threshold
 779  * @tmout: Interrupt aggregation timeout value
 780  */
 781 static inline void
 782 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 783 {
 784         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 785                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 786                       INT_AGGR_TIMEOUT_VAL(tmout),
 787                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 788 }
 789
 790 /**
 791  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 792  * @hba: per adapter instance
 793  */
 794 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 795 {
 796         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 797 }
 798
 799 /**
 800  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 801  *                      When run-stop registers are set to 1, it indicates the
 802  *                      host controller that it can process the requests
 803  * @hba: per adapter instance
 804  */
 805 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 806 {
 807         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 808                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 809         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 810                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 811 }
 812
 813 /**
 814  * ufshcd_hba_start - Start controller initialization sequence
 815  * @hba: per adapter instance
 816  */
 817 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 818 {
 819         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 820 }
 821
 822 /**
 823  * ufshcd_is_hba_active - Get controller state
 824  * @hba: per adapter instance
 825  *
 826  * Returns false if controller is active, true otherwise
 827  */
 828 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 829 {
 830         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 831                 ? false : true;
 832 }
 833
 834 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 835 {
 836         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 837         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
 838             (hba->ufs_version == UFSHCI_VERSION_11))
 839                 return UFS_UNIPRO_VER_1_41;
 840         else
 841                 return UFS_UNIPRO_VER_1_6;
 842 }
 843 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
 844
 845 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 846 {
 847         /*
 848          * If both host and device support UniPro ver1.6 or later, PA layer
 849          * parameters tuning happens during link startup itself.
 850          *
 851          * We can manually tune PA layer parameters if either host or device
 852          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
 853          * logic simple, we will only do manual tuning if local unipro version
 854          * doesn't support ver1.6 or later.
 855          */
 856         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
 857                 return true;
 858         else
 859                 return false;
 860 }
 861
 862 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
 863 {
 864         int ret = 0;
 865         struct ufs_clk_info *clki;
 866         struct list_head *head = &hba->clk_list_head;
 867         ktime_t start = ktime_get();
 868         bool clk_state_changed = false;
 869
 870         if (list_empty(head))
 871                 goto out;
 872
 873         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
 874         if (ret)
 875                 return ret;
 876
 877         list_for_each_entry(clki, head, list) {
 878                 if (!IS_ERR_OR_NULL(clki->clk)) {
 879                         if (scale_up && clki->max_freq) {
 880                                 if (clki->curr_freq == clki->max_freq)
 881                                         continue;
 882
 883                                 clk_state_changed = true;
 884                                 ret = clk_set_rate(clki->clk, clki->max_freq);
 885                                 if (ret) {
 886                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 887                                                 __func__, clki->name,
 888                                                 clki->max_freq, ret);
 889                                         break;
 890                                 }
 891                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
 892                                                 "scaled up", clki->name,
 893                                                 clki->curr_freq,
 894                                                 clki->max_freq);
 895
 896                                 clki->curr_freq = clki->max_freq;
 897
 898                         } else if (!scale_up && clki->min_freq) {
 899                                 if (clki->curr_freq == clki->min_freq)
 900                                         continue;
 901
 902                                 clk_state_changed = true;
 903                                 ret = clk_set_rate(clki->clk, clki->min_freq);
 904                                 if (ret) {
 905                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 906                                                 __func__, clki->name,
 907                                                 clki->min_freq, ret);
 908                                         break;
 909                                 }
 910                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
 911                                                 "scaled down", clki->name,
 912                                                 clki->curr_freq,
 913                                                 clki->min_freq);
 914                                 clki->curr_freq = clki->min_freq;
 915                         }
 916                 }
 917                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
 918                                 clki->name, clk_get_rate(clki->clk));
 919         }
 920
 921         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
 922
 923 out:
 924         if (clk_state_changed)
 925                 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
 926                         (scale_up ? "up" : "down"),
 927                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
 928         return ret;
 929 }
 930
 931 /**
 932  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
 933  * @hba: per adapter instance
 934  * @scale_up: True if scaling up and false if scaling down
 935  *
 936  * Returns true if scaling is required, false otherwise.
 937  */
 938 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
 939                                                bool scale_up)
 940 {
 941         struct ufs_clk_info *clki;
 942         struct list_head *head = &hba->clk_list_head;
 943
 944         if (list_empty(head))
 945                 return false;
 946
 947         list_for_each_entry(clki, head, list) {
 948                 if (!IS_ERR_OR_NULL(clki->clk)) {
 949                         if (scale_up && clki->max_freq) {
 950                                 if (clki->curr_freq == clki->max_freq)
 951                                         continue;
 952                                 return true;
 953                         } else if (!scale_up && clki->min_freq) {
 954                                 if (clki->curr_freq == clki->min_freq)
 955                                         continue;
 956                                 return true;
 957                         }
 958                 }
 959         }
 960
 961         return false;
 962 }
 963
 964 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
 965                                         u64 wait_timeout_us)
 966 {
 967         unsigned long flags;
 968         int ret = 0;
 969         u32 tm_doorbell;
 970         u32 tr_doorbell;
 971         bool timeout = false, do_last_check = false;
 972         ktime_t start;
 973
 974         ufshcd_hold(hba, false);
 975         spin_lock_irqsave(hba->host->host_lock, flags);
 976         /*
 977          * Wait for all the outstanding tasks/transfer requests.
 978          * Verify by checking the doorbell registers are clear.
 979          */
 980         start = ktime_get();
 981         do {
 982                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
 983                         ret = -EBUSY;
 984                         goto out;
 985                 }
 986
 987                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
 988                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 989                 if (!tm_doorbell && !tr_doorbell) {
 990                         timeout = false;
 991                         break;
 992                 } else if (do_last_check) {
 993                         break;
 994                 }
 995
 996                 spin_unlock_irqrestore(hba->host->host_lock, flags);
 997                 schedule();
 998                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
 999                     wait_timeout_us) {
1000                         timeout = true;
1001                         /*
1002                          * We might have scheduled out for long time so make
1003                          * sure to check if doorbells are cleared by this time
1004                          * or not.
1005                          */
1006                         do_last_check = true;
1007                 }
1008                 spin_lock_irqsave(hba->host->host_lock, flags);
1009         } while (tm_doorbell || tr_doorbell);
1010
1011         if (timeout) {
1012                 dev_err(hba->dev,
1013                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1014                         __func__, tm_doorbell, tr_doorbell);
1015                 ret = -EBUSY;
1016         }
1017 out:
1018         spin_unlock_irqrestore(hba->host->host_lock, flags);
1019         ufshcd_release(hba);
1020         return ret;
1021 }
1022
1023 /**
1024  * ufshcd_scale_gear - scale up/down UFS gear
1025  * @hba: per adapter instance
1026  * @scale_up: True for scaling up gear and false for scaling down
1027  *
1028  * Returns 0 for success,
1029  * Returns -EBUSY if scaling can't happen at this time
1030  * Returns non-zero for any other errors
1031  */
1032 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1033 {
1034         #define UFS_MIN_GEAR_TO_SCALE_DOWN      UFS_HS_G1
1035         int ret = 0;
1036         struct ufs_pa_layer_attr new_pwr_info;
1037
1038         if (scale_up) {
1039                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1040                        sizeof(struct ufs_pa_layer_attr));
1041         } else {
1042                 memcpy(&new_pwr_info, &hba->pwr_info,
1043                        sizeof(struct ufs_pa_layer_attr));
1044
1045                 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1046                     || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1047                         /* save the current power mode */
1048                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1049                                 &hba->pwr_info,
1050                                 sizeof(struct ufs_pa_layer_attr));
1051
1052                         /* scale down gear */
1053                         new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1054                         new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1055                 }
1056         }
1057
1058         /* check if the power mode needs to be changed or not? */
1059         ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1060
1061         if (ret)
1062                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1063                         __func__, ret,
1064                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1065                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1066
1067         return ret;
1068 }
1069
1070 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1071 {
1072         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1073         int ret = 0;
1074         /*
1075          * make sure that there are no outstanding requests when
1076          * clock scaling is in progress
1077          */
1078         ufshcd_scsi_block_requests(hba);
1079         down_write(&hba->clk_scaling_lock);
1080         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1081                 ret = -EBUSY;
1082                 up_write(&hba->clk_scaling_lock);
1083                 ufshcd_scsi_unblock_requests(hba);
1084         }
1085
1086         return ret;
1087 }
1088
1089 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1090 {
1091         up_write(&hba->clk_scaling_lock);
1092         ufshcd_scsi_unblock_requests(hba);
1093 }
1094
1095 /**
1096  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1097  * @hba: per adapter instance
1098  * @scale_up: True for scaling up and false for scalin down
1099  *
1100  * Returns 0 for success,
1101  * Returns -EBUSY if scaling can't happen at this time
1102  * Returns non-zero for any other errors
1103  */
1104 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1105 {
1106         int ret = 0;
1107
1108         /* let's not get into low power until clock scaling is completed */
1109         ufshcd_hold(hba, false);
1110
1111         ret = ufshcd_clock_scaling_prepare(hba);
1112         if (ret)
1113                 return ret;
1114
1115         /* scale down the gear before scaling down clocks */
1116         if (!scale_up) {
1117                 ret = ufshcd_scale_gear(hba, false);
1118                 if (ret)
1119                         goto out;
1120         }
1121
1122         ret = ufshcd_scale_clks(hba, scale_up);
1123         if (ret) {
1124                 if (!scale_up)
1125                         ufshcd_scale_gear(hba, true);
1126                 goto out;
1127         }
1128
1129         /* scale up the gear after scaling up clocks */
1130         if (scale_up) {
1131                 ret = ufshcd_scale_gear(hba, true);
1132                 if (ret) {
1133                         ufshcd_scale_clks(hba, false);
1134                         goto out;
1135                 }
1136         }
1137
1138         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1139
1140 out:
1141         ufshcd_clock_scaling_unprepare(hba);
1142         ufshcd_release(hba);
1143         return ret;
1144 }
1145
1146 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1147 {
1148         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1149                                            clk_scaling.suspend_work);
1150         unsigned long irq_flags;
1151
1152         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1153         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1154                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1155                 return;
1156         }
1157         hba->clk_scaling.is_suspended = true;
1158         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1159
1160         __ufshcd_suspend_clkscaling(hba);
1161 }
1162
1163 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1164 {
1165         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1166                                            clk_scaling.resume_work);
1167         unsigned long irq_flags;
1168
1169         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1170         if (!hba->clk_scaling.is_suspended) {
1171                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1172                 return;
1173         }
1174         hba->clk_scaling.is_suspended = false;
1175         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1176
1177         devfreq_resume_device(hba->devfreq);
1178 }
1179
1180 static int ufshcd_devfreq_target(struct device *dev,
1181                                 unsigned long *freq, u32 flags)
1182 {
1183         int ret = 0;
1184         struct ufs_hba *hba = dev_get_drvdata(dev);
1185         ktime_t start;
1186         bool scale_up, sched_clk_scaling_suspend_work = false;
1187         struct list_head *clk_list = &hba->clk_list_head;
1188         struct ufs_clk_info *clki;
1189         unsigned long irq_flags;
1190
1191         if (!ufshcd_is_clkscaling_supported(hba))
1192                 return -EINVAL;
1193
1194         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1195         if (ufshcd_eh_in_progress(hba)) {
1196                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1197                 return 0;
1198         }
1199
1200         if (!hba->clk_scaling.active_reqs)
1201                 sched_clk_scaling_suspend_work = true;
1202
1203         if (list_empty(clk_list)) {
1204                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1205                 goto out;
1206         }
1207
1208         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1209         scale_up = (*freq == clki->max_freq) ? true : false;
1210         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1211                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1212                 ret = 0;
1213                 goto out; /* no state change required */
1214         }
1215         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1216
1217         start = ktime_get();
1218         ret = ufshcd_devfreq_scale(hba, scale_up);
1219
1220         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1221                 (scale_up ? "up" : "down"),
1222                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1223
1224 out:
1225         if (sched_clk_scaling_suspend_work)
1226                 queue_work(hba->clk_scaling.workq,
1227                            &hba->clk_scaling.suspend_work);
1228
1229         return ret;
1230 }
1231
1232 static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1233 {
1234         int *busy = priv;
1235
1236         WARN_ON_ONCE(reserved);
1237         (*busy)++;
1238         return false;
1239 }
1240
1241 /* Whether or not any tag is in use by a request that is in progress. */
1242 static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1243 {
1244         struct request_queue *q = hba->cmd_queue;
1245         int busy = 0;
1246
1247         blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1248         return busy;
1249 }
1250
1251 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1252                 struct devfreq_dev_status *stat)
1253 {
1254         struct ufs_hba *hba = dev_get_drvdata(dev);
1255         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1256         unsigned long flags;
1257
1258         if (!ufshcd_is_clkscaling_supported(hba))
1259                 return -EINVAL;
1260
1261         memset(stat, 0, sizeof(*stat));
1262
1263         spin_lock_irqsave(hba->host->host_lock, flags);
1264         if (!scaling->window_start_t)
1265                 goto start_window;
1266
1267         if (scaling->is_busy_started)
1268                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1269                                         scaling->busy_start_t));
1270
1271         stat->total_time = jiffies_to_usecs((long)jiffies -
1272                                 (long)scaling->window_start_t);
1273         stat->busy_time = scaling->tot_busy_t;
1274 start_window:
1275         scaling->window_start_t = jiffies;
1276         scaling->tot_busy_t = 0;
1277
1278         if (hba->outstanding_reqs) {
1279                 scaling->busy_start_t = ktime_get();
1280                 scaling->is_busy_started = true;
1281         } else {
1282                 scaling->busy_start_t = 0;
1283                 scaling->is_busy_started = false;
1284         }
1285         spin_unlock_irqrestore(hba->host->host_lock, flags);
1286         return 0;
1287 }
1288
1289 static struct devfreq_dev_profile ufs_devfreq_profile = {
1290         .polling_ms     = 100,
1291         .target         = ufshcd_devfreq_target,
1292         .get_dev_status = ufshcd_devfreq_get_dev_status,
1293 };
1294
1295 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1296 {
1297         struct list_head *clk_list = &hba->clk_list_head;
1298         struct ufs_clk_info *clki;
1299         struct devfreq *devfreq;
1300         int ret;
1301
1302         /* Skip devfreq if we don't have any clocks in the list */
1303         if (list_empty(clk_list))
1304                 return 0;
1305
1306         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1307         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1308         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1309
1310         devfreq = devfreq_add_device(hba->dev,
1311                         &ufs_devfreq_profile,
1312                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1313                         NULL);
1314         if (IS_ERR(devfreq)) {
1315                 ret = PTR_ERR(devfreq);
1316                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1317
1318                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1319                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1320                 return ret;
1321         }
1322
1323         hba->devfreq = devfreq;
1324
1325         return 0;
1326 }
1327
1328 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1329 {
1330         struct list_head *clk_list = &hba->clk_list_head;
1331         struct ufs_clk_info *clki;
1332
1333         if (!hba->devfreq)
1334                 return;
1335
1336         devfreq_remove_device(hba->devfreq);
1337         hba->devfreq = NULL;
1338
1339         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1340         dev_pm_opp_remove(hba->dev, clki->min_freq);
1341         dev_pm_opp_remove(hba->dev, clki->max_freq);
1342 }
1343
1344 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1345 {
1346         unsigned long flags;
1347
1348         devfreq_suspend_device(hba->devfreq);
1349         spin_lock_irqsave(hba->host->host_lock, flags);
1350         hba->clk_scaling.window_start_t = 0;
1351         spin_unlock_irqrestore(hba->host->host_lock, flags);
1352 }
1353
1354 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1355 {
1356         unsigned long flags;
1357         bool suspend = false;
1358
1359         if (!ufshcd_is_clkscaling_supported(hba))
1360                 return;
1361
1362         spin_lock_irqsave(hba->host->host_lock, flags);
1363         if (!hba->clk_scaling.is_suspended) {
1364                 suspend = true;
1365                 hba->clk_scaling.is_suspended = true;
1366         }
1367         spin_unlock_irqrestore(hba->host->host_lock, flags);
1368
1369         if (suspend)
1370                 __ufshcd_suspend_clkscaling(hba);
1371 }
1372
1373 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1374 {
1375         unsigned long flags;
1376         bool resume = false;
1377
1378         if (!ufshcd_is_clkscaling_supported(hba))
1379                 return;
1380
1381         spin_lock_irqsave(hba->host->host_lock, flags);
1382         if (hba->clk_scaling.is_suspended) {
1383                 resume = true;
1384                 hba->clk_scaling.is_suspended = false;
1385         }
1386         spin_unlock_irqrestore(hba->host->host_lock, flags);
1387
1388         if (resume)
1389                 devfreq_resume_device(hba->devfreq);
1390 }
1391
1392 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1393                 struct device_attribute *attr, char *buf)
1394 {
1395         struct ufs_hba *hba = dev_get_drvdata(dev);
1396
1397         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1398 }
1399
1400 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1401                 struct device_attribute *attr, const char *buf, size_t count)
1402 {
1403         struct ufs_hba *hba = dev_get_drvdata(dev);
1404         u32 value;
1405         int err;
1406
1407         if (kstrtou32(buf, 0, &value))
1408                 return -EINVAL;
1409
1410         value = !!value;
1411         if (value == hba->clk_scaling.is_allowed)
1412                 goto out;
1413
1414         pm_runtime_get_sync(hba->dev);
1415         ufshcd_hold(hba, false);
1416
1417         cancel_work_sync(&hba->clk_scaling.suspend_work);
1418         cancel_work_sync(&hba->clk_scaling.resume_work);
1419
1420         hba->clk_scaling.is_allowed = value;
1421
1422         if (value) {
1423                 ufshcd_resume_clkscaling(hba);
1424         } else {
1425                 ufshcd_suspend_clkscaling(hba);
1426                 err = ufshcd_devfreq_scale(hba, true);
1427                 if (err)
1428                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1429                                         __func__, err);
1430         }
1431
1432         ufshcd_release(hba);
1433         pm_runtime_put_sync(hba->dev);
1434 out:
1435         return count;
1436 }
1437
1438 static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1439 {
1440         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1441         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1442         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1443         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1444         hba->clk_scaling.enable_attr.attr.mode = 0644;
1445         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1446                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1447 }
1448
1449 static void ufshcd_ungate_work(struct work_struct *work)
1450 {
1451         int ret;
1452         unsigned long flags;
1453         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1454                         clk_gating.ungate_work);
1455
1456         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1457
1458         spin_lock_irqsave(hba->host->host_lock, flags);
1459         if (hba->clk_gating.state == CLKS_ON) {
1460                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1461                 goto unblock_reqs;
1462         }
1463
1464         spin_unlock_irqrestore(hba->host->host_lock, flags);
1465         ufshcd_setup_clocks(hba, true);
1466
1467         ufshcd_enable_irq(hba);
1468
1469         /* Exit from hibern8 */
1470         if (ufshcd_can_hibern8_during_gating(hba)) {
1471                 /* Prevent gating in this path */
1472                 hba->clk_gating.is_suspended = true;
1473                 if (ufshcd_is_link_hibern8(hba)) {
1474                         ret = ufshcd_uic_hibern8_exit(hba);
1475                         if (ret)
1476                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1477                                         __func__, ret);
1478                         else
1479                                 ufshcd_set_link_active(hba);
1480                 }
1481                 hba->clk_gating.is_suspended = false;
1482         }
1483 unblock_reqs:
1484         ufshcd_scsi_unblock_requests(hba);
1485 }
1486
1487 /**
1488  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1489  * Also, exit from hibern8 mode and set the link as active.
1490  * @hba: per adapter instance
1491  * @async: This indicates whether caller should ungate clocks asynchronously.
1492  */
1493 int ufshcd_hold(struct ufs_hba *hba, bool async)
1494 {
1495         int rc = 0;
1496         unsigned long flags;
1497
1498         if (!ufshcd_is_clkgating_allowed(hba))
1499                 goto out;
1500         spin_lock_irqsave(hba->host->host_lock, flags);
1501         hba->clk_gating.active_reqs++;
1502
1503         if (ufshcd_eh_in_progress(hba)) {
1504                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1505                 return 0;
1506         }
1507
1508 start:
1509         switch (hba->clk_gating.state) {
1510         case CLKS_ON:
1511                 /*
1512                  * Wait for the ungate work to complete if in progress.
1513                  * Though the clocks may be in ON state, the link could
1514                  * still be in hibner8 state if hibern8 is allowed
1515                  * during clock gating.
1516                  * Make sure we exit hibern8 state also in addition to
1517                  * clocks being ON.
1518                  */
1519                 if (ufshcd_can_hibern8_during_gating(hba) &&
1520                     ufshcd_is_link_hibern8(hba)) {
1521                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1522                         flush_work(&hba->clk_gating.ungate_work);
1523                         spin_lock_irqsave(hba->host->host_lock, flags);
1524                         goto start;
1525                 }
1526                 break;
1527         case REQ_CLKS_OFF:
1528                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1529                         hba->clk_gating.state = CLKS_ON;
1530                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1531                                                 hba->clk_gating.state);
1532                         break;
1533                 }
1534                 /*
1535                  * If we are here, it means gating work is either done or
1536                  * currently running. Hence, fall through to cancel gating
1537                  * work and to enable clocks.
1538                  */
1539                 /* fallthrough */
1540         case CLKS_OFF:
1541                 ufshcd_scsi_block_requests(hba);
1542                 hba->clk_gating.state = REQ_CLKS_ON;
1543                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1544                                         hba->clk_gating.state);
1545                 queue_work(hba->clk_gating.clk_gating_workq,
1546                            &hba->clk_gating.ungate_work);
1547                 /*
1548                  * fall through to check if we should wait for this
1549                  * work to be done or not.
1550                  */
1551                 /* fallthrough */
1552         case REQ_CLKS_ON:
1553                 if (async) {
1554                         rc = -EAGAIN;
1555                         hba->clk_gating.active_reqs--;
1556                         break;
1557                 }
1558
1559                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1560                 flush_work(&hba->clk_gating.ungate_work);
1561                 /* Make sure state is CLKS_ON before returning */
1562                 spin_lock_irqsave(hba->host->host_lock, flags);
1563                 goto start;
1564         default:
1565                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1566                                 __func__, hba->clk_gating.state);
1567                 break;
1568         }
1569         spin_unlock_irqrestore(hba->host->host_lock, flags);
1570 out:
1571         return rc;
1572 }
1573 EXPORT_SYMBOL_GPL(ufshcd_hold);
1574
1575 static void ufshcd_gate_work(struct work_struct *work)
1576 {
1577         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1578                         clk_gating.gate_work.work);
1579         unsigned long flags;
1580
1581         spin_lock_irqsave(hba->host->host_lock, flags);
1582         /*
1583          * In case you are here to cancel this work the gating state
1584          * would be marked as REQ_CLKS_ON. In this case save time by
1585          * skipping the gating work and exit after changing the clock
1586          * state to CLKS_ON.
1587          */
1588         if (hba->clk_gating.is_suspended ||
1589                 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1590                 hba->clk_gating.state = CLKS_ON;
1591                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1592                                         hba->clk_gating.state);
1593                 goto rel_lock;
1594         }
1595
1596         if (hba->clk_gating.active_reqs
1597                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1598                 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1599                 || hba->active_uic_cmd || hba->uic_async_done)
1600                 goto rel_lock;
1601
1602         spin_unlock_irqrestore(hba->host->host_lock, flags);
1603
1604         /* put the link into hibern8 mode before turning off clocks */
1605         if (ufshcd_can_hibern8_during_gating(hba)) {
1606                 if (ufshcd_uic_hibern8_enter(hba)) {
1607                         hba->clk_gating.state = CLKS_ON;
1608                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1609                                                 hba->clk_gating.state);
1610                         goto out;
1611                 }
1612                 ufshcd_set_link_hibern8(hba);
1613         }
1614
1615         ufshcd_disable_irq(hba);
1616
1617         if (!ufshcd_is_link_active(hba))
1618                 ufshcd_setup_clocks(hba, false);
1619         else
1620                 /* If link is active, device ref_clk can't be switched off */
1621                 __ufshcd_setup_clocks(hba, false, true);
1622
1623         /*
1624          * In case you are here to cancel this work the gating state
1625          * would be marked as REQ_CLKS_ON. In this case keep the state
1626          * as REQ_CLKS_ON which would anyway imply that clocks are off
1627          * and a request to turn them on is pending. By doing this way,
1628          * we keep the state machine in tact and this would ultimately
1629          * prevent from doing cancel work multiple times when there are
1630          * new requests arriving before the current cancel work is done.
1631          */
1632         spin_lock_irqsave(hba->host->host_lock, flags);
1633         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1634                 hba->clk_gating.state = CLKS_OFF;
1635                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1636                                         hba->clk_gating.state);
1637         }
1638 rel_lock:
1639         spin_unlock_irqrestore(hba->host->host_lock, flags);
1640 out:
1641         return;
1642 }
1643
1644 /* host lock must be held before calling this variant */
1645 static void __ufshcd_release(struct ufs_hba *hba)
1646 {
1647         if (!ufshcd_is_clkgating_allowed(hba))
1648                 return;
1649
1650         hba->clk_gating.active_reqs--;
1651
1652         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1653                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1654                 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1655                 || hba->active_uic_cmd || hba->uic_async_done
1656                 || ufshcd_eh_in_progress(hba))
1657                 return;
1658
1659         hba->clk_gating.state = REQ_CLKS_OFF;
1660         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1661         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1662                            &hba->clk_gating.gate_work,
1663                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1664 }
1665
1666 void ufshcd_release(struct ufs_hba *hba)
1667 {
1668         unsigned long flags;
1669
1670         spin_lock_irqsave(hba->host->host_lock, flags);
1671         __ufshcd_release(hba);
1672         spin_unlock_irqrestore(hba->host->host_lock, flags);
1673 }
1674 EXPORT_SYMBOL_GPL(ufshcd_release);
1675
1676 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1677                 struct device_attribute *attr, char *buf)
1678 {
1679         struct ufs_hba *hba = dev_get_drvdata(dev);
1680
1681         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1682 }
1683
1684 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1685                 struct device_attribute *attr, const char *buf, size_t count)
1686 {
1687         struct ufs_hba *hba = dev_get_drvdata(dev);
1688         unsigned long flags, value;
1689
1690         if (kstrtoul(buf, 0, &value))
1691                 return -EINVAL;
1692
1693         spin_lock_irqsave(hba->host->host_lock, flags);
1694         hba->clk_gating.delay_ms = value;
1695         spin_unlock_irqrestore(hba->host->host_lock, flags);
1696         return count;
1697 }
1698
1699 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1700                 struct device_attribute *attr, char *buf)
1701 {
1702         struct ufs_hba *hba = dev_get_drvdata(dev);
1703
1704         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1705 }
1706
1707 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1708                 struct device_attribute *attr, const char *buf, size_t count)
1709 {
1710         struct ufs_hba *hba = dev_get_drvdata(dev);
1711         unsigned long flags;
1712         u32 value;
1713
1714         if (kstrtou32(buf, 0, &value))
1715                 return -EINVAL;
1716
1717         value = !!value;
1718         if (value == hba->clk_gating.is_enabled)
1719                 goto out;
1720
1721         if (value) {
1722                 ufshcd_release(hba);
1723         } else {
1724                 spin_lock_irqsave(hba->host->host_lock, flags);
1725                 hba->clk_gating.active_reqs++;
1726                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1727         }
1728
1729         hba->clk_gating.is_enabled = value;
1730 out:
1731         return count;
1732 }
1733
1734 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1735 {
1736         char wq_name[sizeof("ufs_clkscaling_00")];
1737
1738         if (!ufshcd_is_clkscaling_supported(hba))
1739                 return;
1740
1741         INIT_WORK(&hba->clk_scaling.suspend_work,
1742                   ufshcd_clk_scaling_suspend_work);
1743         INIT_WORK(&hba->clk_scaling.resume_work,
1744                   ufshcd_clk_scaling_resume_work);
1745
1746         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1747                  hba->host->host_no);
1748         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1749
1750         ufshcd_clkscaling_init_sysfs(hba);
1751 }
1752
1753 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1754 {
1755         if (!ufshcd_is_clkscaling_supported(hba))
1756                 return;
1757
1758         destroy_workqueue(hba->clk_scaling.workq);
1759         ufshcd_devfreq_remove(hba);
1760 }
1761
1762 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1763 {
1764         char wq_name[sizeof("ufs_clk_gating_00")];
1765
1766         if (!ufshcd_is_clkgating_allowed(hba))
1767                 return;
1768
1769         hba->clk_gating.delay_ms = 150;
1770         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1771         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1772
1773         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1774                  hba->host->host_no);
1775         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1776                                                            WQ_MEM_RECLAIM);
1777
1778         hba->clk_gating.is_enabled = true;
1779
1780         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1781         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1782         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1783         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1784         hba->clk_gating.delay_attr.attr.mode = 0644;
1785         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1786                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1787
1788         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1789         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1790         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1791         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1792         hba->clk_gating.enable_attr.attr.mode = 0644;
1793         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1794                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1795 }
1796
1797 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1798 {
1799         if (!ufshcd_is_clkgating_allowed(hba))
1800                 return;
1801         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1802         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1803         cancel_work_sync(&hba->clk_gating.ungate_work);
1804         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1805         destroy_workqueue(hba->clk_gating.clk_gating_workq);
1806 }
1807
1808 /* Must be called with host lock acquired */
1809 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1810 {
1811         bool queue_resume_work = false;
1812
1813         if (!ufshcd_is_clkscaling_supported(hba))
1814                 return;
1815
1816         if (!hba->clk_scaling.active_reqs++)
1817                 queue_resume_work = true;
1818
1819         if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1820                 return;
1821
1822         if (queue_resume_work)
1823                 queue_work(hba->clk_scaling.workq,
1824                            &hba->clk_scaling.resume_work);
1825
1826         if (!hba->clk_scaling.window_start_t) {
1827                 hba->clk_scaling.window_start_t = jiffies;
1828                 hba->clk_scaling.tot_busy_t = 0;
1829                 hba->clk_scaling.is_busy_started = false;
1830         }
1831
1832         if (!hba->clk_scaling.is_busy_started) {
1833                 hba->clk_scaling.busy_start_t = ktime_get();
1834                 hba->clk_scaling.is_busy_started = true;
1835         }
1836 }
1837
1838 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1839 {
1840         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1841
1842         if (!ufshcd_is_clkscaling_supported(hba))
1843                 return;
1844
1845         if (!hba->outstanding_reqs && scaling->is_busy_started) {
1846                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1847                                         scaling->busy_start_t));
1848                 scaling->busy_start_t = 0;
1849                 scaling->is_busy_started = false;
1850         }
1851 }
1852 /**
1853  * ufshcd_send_command - Send SCSI or device management commands
1854  * @hba: per adapter instance
1855  * @task_tag: Task tag of the command
1856  */
1857 static inline
1858 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1859 {
1860         hba->lrb[task_tag].issue_time_stamp = ktime_get();
1861         hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
1862         ufshcd_add_command_trace(hba, task_tag, "send");
1863         ufshcd_clk_scaling_start_busy(hba);
1864         __set_bit(task_tag, &hba->outstanding_reqs);
1865         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1866         /* Make sure that doorbell is committed immediately */
1867         wmb();
1868 }
1869
1870 /**
1871  * ufshcd_copy_sense_data - Copy sense data in case of check condition
1872  * @lrbp: pointer to local reference block
1873  */
1874 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1875 {
1876         int len;
1877         if (lrbp->sense_buffer &&
1878             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1879                 int len_to_copy;
1880
1881                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1882                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
1883
1884                 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1885                        len_to_copy);
1886         }
1887 }
1888
1889 /**
1890  * ufshcd_copy_query_response() - Copy the Query Response and the data
1891  * descriptor
1892  * @hba: per adapter instance
1893  * @lrbp: pointer to local reference block
1894  */
1895 static
1896 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1897 {
1898         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1899
1900         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
1901
1902         /* Get the descriptor */
1903         if (hba->dev_cmd.query.descriptor &&
1904             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
1905                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
1906                                 GENERAL_UPIU_REQUEST_SIZE;
1907                 u16 resp_len;
1908                 u16 buf_len;
1909
1910                 /* data segment length */
1911                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
1912                                                 MASK_QUERY_DATA_SEG_LEN;
1913                 buf_len = be16_to_cpu(
1914                                 hba->dev_cmd.query.request.upiu_req.length);
1915                 if (likely(buf_len >= resp_len)) {
1916                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1917                 } else {
1918                         dev_warn(hba->dev,
1919                                  "%s: rsp size %d is bigger than buffer size %d",
1920                                  __func__, resp_len, buf_len);
1921                         return -EINVAL;
1922                 }
1923         }
1924
1925         return 0;
1926 }
1927
1928 /**
1929  * ufshcd_hba_capabilities - Read controller capabilities
1930  * @hba: per adapter instance
1931  */
1932 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1933 {
1934         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1935
1936         /* nutrs and nutmrs are 0 based values */
1937         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1938         hba->nutmrs =
1939         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1940 }
1941
1942 /**
1943  * ufshcd_ready_for_uic_cmd - Check if controller is ready
1944  *                            to accept UIC commands
1945  * @hba: per adapter instance
1946  * Return true on success, else false
1947  */
1948 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1949 {
1950         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1951                 return true;
1952         else
1953                 return false;
1954 }
1955
1956 /**
1957  * ufshcd_get_upmcrs - Get the power mode change request status
1958  * @hba: Pointer to adapter instance
1959  *
1960  * This function gets the UPMCRS field of HCS register
1961  * Returns value of UPMCRS field
1962  */
1963 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1964 {
1965         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1966 }
1967
1968 /**
1969  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1970  * @hba: per adapter instance
1971  * @uic_cmd: UIC command
1972  *
1973  * Mutex must be held.
1974  */
1975 static inline void
1976 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1977 {
1978         WARN_ON(hba->active_uic_cmd);
1979
1980         hba->active_uic_cmd = uic_cmd;
1981
1982         /* Write Args */
1983         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1984         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1985         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
1986
1987         /* Write UIC Cmd */
1988         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
1989                       REG_UIC_COMMAND);
1990 }
1991
1992 /**
1993  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
1994  * @hba: per adapter instance
1995  * @uic_cmd: UIC command
1996  *
1997  * Must be called with mutex held.
1998  * Returns 0 only if success.
1999  */
2000 static int
2001 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2002 {
2003         int ret;
2004         unsigned long flags;
2005
2006         if (wait_for_completion_timeout(&uic_cmd->done,
2007                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2008                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2009         else
2010                 ret = -ETIMEDOUT;
2011
2012         spin_lock_irqsave(hba->host->host_lock, flags);
2013         hba->active_uic_cmd = NULL;
2014         spin_unlock_irqrestore(hba->host->host_lock, flags);
2015
2016         return ret;
2017 }
2018
2019 /**
2020  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2021  * @hba: per adapter instance
2022  * @uic_cmd: UIC command
2023  * @completion: initialize the completion only if this is set to true
2024  *
2025  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2026  * with mutex held and host_lock locked.
2027  * Returns 0 only if success.
2028  */
2029 static int
2030 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2031                       bool completion)
2032 {
2033         if (!ufshcd_ready_for_uic_cmd(hba)) {
2034                 dev_err(hba->dev,
2035                         "Controller not ready to accept UIC commands\n");
2036                 return -EIO;
2037         }
2038
2039         if (completion)
2040                 init_completion(&uic_cmd->done);
2041
2042         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2043
2044         return 0;
2045 }
2046
2047 /**
2048  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2049  * @hba: per adapter instance
2050  * @uic_cmd: UIC command
2051  *
2052  * Returns 0 only if success.
2053  */
2054 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2055 {
2056         int ret;
2057         unsigned long flags;
2058
2059         ufshcd_hold(hba, false);
2060         mutex_lock(&hba->uic_cmd_mutex);
2061         ufshcd_add_delay_before_dme_cmd(hba);
2062
2063         spin_lock_irqsave(hba->host->host_lock, flags);
2064         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2065         spin_unlock_irqrestore(hba->host->host_lock, flags);
2066         if (!ret)
2067                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2068
2069         mutex_unlock(&hba->uic_cmd_mutex);
2070
2071         ufshcd_release(hba);
2072         return ret;
2073 }
2074
2075 /**
2076  * ufshcd_map_sg - Map scatter-gather list to prdt
2077  * @hba: per adapter instance
2078  * @lrbp: pointer to local reference block
2079  *
2080  * Returns 0 in case of success, non-zero value in case of failure
2081  */
2082 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2083 {
2084         struct ufshcd_sg_entry *prd_table;
2085         struct scatterlist *sg;
2086         struct scsi_cmnd *cmd;
2087         int sg_segments;
2088         int i;
2089
2090         cmd = lrbp->cmd;
2091         sg_segments = scsi_dma_map(cmd);
2092         if (sg_segments < 0)
2093                 return sg_segments;
2094
2095         if (sg_segments) {
2096                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2097                         lrbp->utr_descriptor_ptr->prd_table_length =
2098                                 cpu_to_le16((u16)(sg_segments *
2099                                         sizeof(struct ufshcd_sg_entry)));
2100                 else
2101                         lrbp->utr_descriptor_ptr->prd_table_length =
2102                                 cpu_to_le16((u16) (sg_segments));
2103
2104                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2105
2106                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2107                         prd_table[i].size  =
2108                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2109                         prd_table[i].base_addr =
2110                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2111                         prd_table[i].upper_addr =
2112                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2113                         prd_table[i].reserved = 0;
2114                 }
2115         } else {
2116                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2117         }
2118
2119         return 0;
2120 }
2121
2122 /**
2123  * ufshcd_enable_intr - enable interrupts
2124  * @hba: per adapter instance
2125  * @intrs: interrupt bits
2126  */
2127 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2128 {
2129         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2130
2131         if (hba->ufs_version == UFSHCI_VERSION_10) {
2132                 u32 rw;
2133                 rw = set & INTERRUPT_MASK_RW_VER_10;
2134                 set = rw | ((set ^ intrs) & intrs);
2135         } else {
2136                 set |= intrs;
2137         }
2138
2139         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2140 }
2141
2142 /**
2143  * ufshcd_disable_intr - disable interrupts
2144  * @hba: per adapter instance
2145  * @intrs: interrupt bits
2146  */
2147 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2148 {
2149         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2150
2151         if (hba->ufs_version == UFSHCI_VERSION_10) {
2152                 u32 rw;
2153                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2154                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2155                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2156
2157         } else {
2158                 set &= ~intrs;
2159         }
2160
2161         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2162 }
2163
2164 /**
2165  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2166  * descriptor according to request
2167  * @lrbp: pointer to local reference block
2168  * @upiu_flags: flags required in the header
2169  * @cmd_dir: requests data direction
2170  */
2171 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2172                         u32 *upiu_flags, enum dma_data_direction cmd_dir)
2173 {
2174         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2175         u32 data_direction;
2176         u32 dword_0;
2177
2178         if (cmd_dir == DMA_FROM_DEVICE) {
2179                 data_direction = UTP_DEVICE_TO_HOST;
2180                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2181         } else if (cmd_dir == DMA_TO_DEVICE) {
2182                 data_direction = UTP_HOST_TO_DEVICE;
2183                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2184         } else {
2185                 data_direction = UTP_NO_DATA_TRANSFER;
2186                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2187         }
2188
2189         dword_0 = data_direction | (lrbp->command_type
2190                                 << UPIU_COMMAND_TYPE_OFFSET);
2191         if (lrbp->intr_cmd)
2192                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2193
2194         /* Transfer request descriptor header fields */
2195         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2196         /* dword_1 is reserved, hence it is set to 0 */
2197         req_desc->header.dword_1 = 0;
2198         /*
2199          * assigning invalid value for command status. Controller
2200          * updates OCS on command completion, with the command
2201          * status
2202          */
2203         req_desc->header.dword_2 =
2204                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2205         /* dword_3 is reserved, hence it is set to 0 */
2206         req_desc->header.dword_3 = 0;
2207
2208         req_desc->prd_table_length = 0;
2209 }
2210
2211 /**
2212  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2213  * for scsi commands
2214  * @lrbp: local reference block pointer
2215  * @upiu_flags: flags
2216  */
2217 static
2218 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2219 {
2220         struct scsi_cmnd *cmd = lrbp->cmd;
2221         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2222         unsigned short cdb_len;
2223
2224         /* command descriptor fields */
2225         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2226                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2227                                 lrbp->lun, lrbp->task_tag);
2228         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2229                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2230
2231         /* Total EHS length and Data segment length will be zero */
2232         ucd_req_ptr->header.dword_2 = 0;
2233
2234         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2235
2236         cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2237         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2238         memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2239
2240         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2241 }
2242
2243 /**
2244  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2245  * for query requsts
2246  * @hba: UFS hba
2247  * @lrbp: local reference block pointer
2248  * @upiu_flags: flags
2249  */
2250 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2251                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2252 {
2253         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2254         struct ufs_query *query = &hba->dev_cmd.query;
2255         u16 len = be16_to_cpu(query->request.upiu_req.length);
2256
2257         /* Query request header */
2258         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2259                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2260                         lrbp->lun, lrbp->task_tag);
2261         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2262                         0, query->request.query_func, 0, 0);
2263
2264         /* Data segment length only need for WRITE_DESC */
2265         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2266                 ucd_req_ptr->header.dword_2 =
2267                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2268         else
2269                 ucd_req_ptr->header.dword_2 = 0;
2270
2271         /* Copy the Query Request buffer as is */
2272         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2273                         QUERY_OSF_SIZE);
2274
2275         /* Copy the Descriptor */
2276         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2277                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2278
2279         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2280 }
2281
2282 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2283 {
2284         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2285
2286         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2287
2288         /* command descriptor fields */
2289         ucd_req_ptr->header.dword_0 =
2290                 UPIU_HEADER_DWORD(
2291                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2292         /* clear rest of the fields of basic header */
2293         ucd_req_ptr->header.dword_1 = 0;
2294         ucd_req_ptr->header.dword_2 = 0;
2295
2296         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2297 }
2298
2299 /**
2300  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2301  *                           for Device Management Purposes
2302  * @hba: per adapter instance
2303  * @lrbp: pointer to local reference block
2304  */
2305 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2306 {
2307         u32 upiu_flags;
2308         int ret = 0;
2309
2310         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2311             (hba->ufs_version == UFSHCI_VERSION_11))
2312                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2313         else
2314                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2315
2316         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2317         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2318                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2319         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2320                 ufshcd_prepare_utp_nop_upiu(lrbp);
2321         else
2322                 ret = -EINVAL;
2323
2324         return ret;
2325 }
2326
2327 /**
2328  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2329  *                         for SCSI Purposes
2330  * @hba: per adapter instance
2331  * @lrbp: pointer to local reference block
2332  */
2333 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2334 {
2335         u32 upiu_flags;
2336         int ret = 0;
2337
2338         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2339             (hba->ufs_version == UFSHCI_VERSION_11))
2340                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2341         else
2342                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2343
2344         if (likely(lrbp->cmd)) {
2345                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2346                                                 lrbp->cmd->sc_data_direction);
2347                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2348         } else {
2349                 ret = -EINVAL;
2350         }
2351
2352         return ret;
2353 }
2354
2355 /**
2356  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2357  * @upiu_wlun_id: UPIU W-LUN id
2358  *
2359  * Returns SCSI W-LUN id
2360  */
2361 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2362 {
2363         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2364 }
2365
2366 /**
2367  * ufshcd_queuecommand - main entry point for SCSI requests
2368  * @host: SCSI host pointer
2369  * @cmd: command from SCSI Midlayer
2370  *
2371  * Returns 0 for success, non-zero in case of failure
2372  */
2373 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2374 {
2375         struct ufshcd_lrb *lrbp;
2376         struct ufs_hba *hba;
2377         unsigned long flags;
2378         int tag;
2379         int err = 0;
2380
2381         hba = shost_priv(host);
2382
2383         tag = cmd->request->tag;
2384         if (!ufshcd_valid_tag(hba, tag)) {
2385                 dev_err(hba->dev,
2386                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2387                         __func__, tag, cmd, cmd->request);
2388                 BUG();
2389         }
2390
2391         if (!down_read_trylock(&hba->clk_scaling_lock))
2392                 return SCSI_MLQUEUE_HOST_BUSY;
2393
2394         spin_lock_irqsave(hba->host->host_lock, flags);
2395         switch (hba->ufshcd_state) {
2396         case UFSHCD_STATE_OPERATIONAL:
2397                 break;
2398         case UFSHCD_STATE_EH_SCHEDULED:
2399         case UFSHCD_STATE_RESET:
2400                 err = SCSI_MLQUEUE_HOST_BUSY;
2401                 goto out_unlock;
2402         case UFSHCD_STATE_ERROR:
2403                 set_host_byte(cmd, DID_ERROR);
2404                 cmd->scsi_done(cmd);
2405                 goto out_unlock;
2406         default:
2407                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2408                                 __func__, hba->ufshcd_state);
2409                 set_host_byte(cmd, DID_BAD_TARGET);
2410                 cmd->scsi_done(cmd);
2411                 goto out_unlock;
2412         }
2413
2414         /* if error handling is in progress, don't issue commands */
2415         if (ufshcd_eh_in_progress(hba)) {
2416                 set_host_byte(cmd, DID_ERROR);
2417                 cmd->scsi_done(cmd);
2418                 goto out_unlock;
2419         }
2420         spin_unlock_irqrestore(hba->host->host_lock, flags);
2421
2422         hba->req_abort_count = 0;
2423
2424         err = ufshcd_hold(hba, true);
2425         if (err) {
2426                 err = SCSI_MLQUEUE_HOST_BUSY;
2427                 goto out;
2428         }
2429         WARN_ON(hba->clk_gating.state != CLKS_ON);
2430
2431         lrbp = &hba->lrb[tag];
2432
2433         WARN_ON(lrbp->cmd);
2434         lrbp->cmd = cmd;
2435         lrbp->sense_bufflen = UFS_SENSE_SIZE;
2436         lrbp->sense_buffer = cmd->sense_buffer;
2437         lrbp->task_tag = tag;
2438         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2439         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2440         lrbp->req_abort_skip = false;
2441
2442         ufshcd_comp_scsi_upiu(hba, lrbp);
2443
2444         err = ufshcd_map_sg(hba, lrbp);
2445         if (err) {
2446                 lrbp->cmd = NULL;
2447                 ufshcd_release(hba);
2448                 goto out;
2449         }
2450         /* Make sure descriptors are ready before ringing the doorbell */
2451         wmb();
2452
2453         /* issue command to the controller */
2454         spin_lock_irqsave(hba->host->host_lock, flags);
2455         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2456         ufshcd_send_command(hba, tag);
2457 out_unlock:
2458         spin_unlock_irqrestore(hba->host->host_lock, flags);
2459 out:
2460         up_read(&hba->clk_scaling_lock);
2461         return err;
2462 }
2463
2464 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2465                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2466 {
2467         lrbp->cmd = NULL;
2468         lrbp->sense_bufflen = 0;
2469         lrbp->sense_buffer = NULL;
2470         lrbp->task_tag = tag;
2471         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2472         lrbp->intr_cmd = true; /* No interrupt aggregation */
2473         hba->dev_cmd.type = cmd_type;
2474
2475         return ufshcd_comp_devman_upiu(hba, lrbp);
2476 }
2477
2478 static int
2479 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2480 {
2481         int err = 0;
2482         unsigned long flags;
2483         u32 mask = 1 << tag;
2484
2485         /* clear outstanding transaction before retry */
2486         spin_lock_irqsave(hba->host->host_lock, flags);
2487         ufshcd_utrl_clear(hba, tag);
2488         spin_unlock_irqrestore(hba->host->host_lock, flags);
2489
2490         /*
2491          * wait for for h/w to clear corresponding bit in door-bell.
2492          * max. wait is 1 sec.
2493          */
2494         err = ufshcd_wait_for_register(hba,
2495                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2496                         mask, ~mask, 1000, 1000, true);
2497
2498         return err;
2499 }
2500
2501 static int
2502 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2503 {
2504         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2505
2506         /* Get the UPIU response */
2507         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2508                                 UPIU_RSP_CODE_OFFSET;
2509         return query_res->response;
2510 }
2511
2512 /**
2513  * ufshcd_dev_cmd_completion() - handles device management command responses
2514  * @hba: per adapter instance
2515  * @lrbp: pointer to local reference block
2516  */
2517 static int
2518 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2519 {
2520         int resp;
2521         int err = 0;
2522
2523         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2524         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2525
2526         switch (resp) {
2527         case UPIU_TRANSACTION_NOP_IN:
2528                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2529                         err = -EINVAL;
2530                         dev_err(hba->dev, "%s: unexpected response %x\n",
2531                                         __func__, resp);
2532                 }
2533                 break;
2534         case UPIU_TRANSACTION_QUERY_RSP:
2535                 err = ufshcd_check_query_response(hba, lrbp);
2536                 if (!err)
2537                         err = ufshcd_copy_query_response(hba, lrbp);
2538                 break;
2539         case UPIU_TRANSACTION_REJECT_UPIU:
2540                 /* TODO: handle Reject UPIU Response */
2541                 err = -EPERM;
2542                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2543                                 __func__);
2544                 break;
2545         default:
2546                 err = -EINVAL;
2547                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2548                                 __func__, resp);
2549                 break;
2550         }
2551
2552         return err;
2553 }
2554
2555 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2556                 struct ufshcd_lrb *lrbp, int max_timeout)
2557 {
2558         int err = 0;
2559         unsigned long time_left;
2560         unsigned long flags;
2561
2562         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2563                         msecs_to_jiffies(max_timeout));
2564
2565         /* Make sure descriptors are ready before ringing the doorbell */
2566         wmb();
2567         spin_lock_irqsave(hba->host->host_lock, flags);
2568         hba->dev_cmd.complete = NULL;
2569         if (likely(time_left)) {
2570                 err = ufshcd_get_tr_ocs(lrbp);
2571                 if (!err)
2572                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2573         }
2574         spin_unlock_irqrestore(hba->host->host_lock, flags);
2575
2576         if (!time_left) {
2577                 err = -ETIMEDOUT;
2578                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2579                         __func__, lrbp->task_tag);
2580                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2581                         /* successfully cleared the command, retry if needed */
2582                         err = -EAGAIN;
2583                 /*
2584                  * in case of an error, after clearing the doorbell,
2585                  * we also need to clear the outstanding_request
2586                  * field in hba
2587                  */
2588                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2589         }
2590
2591         return err;
2592 }
2593
2594 /**
2595  * ufshcd_exec_dev_cmd - API for sending device management requests
2596  * @hba: UFS hba
2597  * @cmd_type: specifies the type (NOP, Query...)
2598  * @timeout: time in seconds
2599  *
2600  * NOTE: Since there is only one available tag for device management commands,
2601  * it is expected you hold the hba->dev_cmd.lock mutex.
2602  */
2603 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2604                 enum dev_cmd_type cmd_type, int timeout)
2605 {
2606         struct request_queue *q = hba->cmd_queue;
2607         struct request *req;
2608         struct ufshcd_lrb *lrbp;
2609         int err;
2610         int tag;
2611         struct completion wait;
2612         unsigned long flags;
2613
2614         down_read(&hba->clk_scaling_lock);
2615
2616         /*
2617          * Get free slot, sleep if slots are unavailable.
2618          * Even though we use wait_event() which sleeps indefinitely,
2619          * the maximum wait time is bounded by SCSI request timeout.
2620          */
2621         req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2622         if (IS_ERR(req)) {
2623                 err = PTR_ERR(req);
2624                 goto out_unlock;
2625         }
2626         tag = req->tag;
2627         WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
2628
2629         init_completion(&wait);
2630         lrbp = &hba->lrb[tag];
2631         WARN_ON(lrbp->cmd);
2632         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2633         if (unlikely(err))
2634                 goto out_put_tag;
2635
2636         hba->dev_cmd.complete = &wait;
2637
2638         ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2639         /* Make sure descriptors are ready before ringing the doorbell */
2640         wmb();
2641         spin_lock_irqsave(hba->host->host_lock, flags);
2642         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2643         ufshcd_send_command(hba, tag);
2644         spin_unlock_irqrestore(hba->host->host_lock, flags);
2645
2646         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2647
2648         ufshcd_add_query_upiu_trace(hba, tag,
2649                         err ? "query_complete_err" : "query_complete");
2650
2651 out_put_tag:
2652         blk_put_request(req);
2653 out_unlock:
2654         up_read(&hba->clk_scaling_lock);
2655         return err;
2656 }
2657
2658 /**
2659  * ufshcd_init_query() - init the query response and request parameters
2660  * @hba: per-adapter instance
2661  * @request: address of the request pointer to be initialized
2662  * @response: address of the response pointer to be initialized
2663  * @opcode: operation to perform
2664  * @idn: flag idn to access
2665  * @index: LU number to access
2666  * @selector: query/flag/descriptor further identification
2667  */
2668 static inline void ufshcd_init_query(struct ufs_hba *hba,
2669                 struct ufs_query_req **request, struct ufs_query_res **response,
2670                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2671 {
2672         *request = &hba->dev_cmd.query.request;
2673         *response = &hba->dev_cmd.query.response;
2674         memset(*request, 0, sizeof(struct ufs_query_req));
2675         memset(*response, 0, sizeof(struct ufs_query_res));
2676         (*request)->upiu_req.opcode = opcode;
2677         (*request)->upiu_req.idn = idn;
2678         (*request)->upiu_req.index = index;
2679         (*request)->upiu_req.selector = selector;
2680 }
2681
2682 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2683         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2684 {
2685         int ret;
2686         int retries;
2687
2688         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2689                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2690                 if (ret)
2691                         dev_dbg(hba->dev,
2692                                 "%s: failed with error %d, retries %d\n",
2693                                 __func__, ret, retries);
2694                 else
2695                         break;
2696         }
2697
2698         if (ret)
2699                 dev_err(hba->dev,
2700                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2701                         __func__, opcode, idn, ret, retries);
2702         return ret;
2703 }
2704
2705 /**
2706  * ufshcd_query_flag() - API function for sending flag query requests
2707  * @hba: per-adapter instance
2708  * @opcode: flag query to perform
2709  * @idn: flag idn to access
2710  * @flag_res: the flag value after the query request completes
2711  *
2712  * Returns 0 for success, non-zero in case of failure
2713  */
2714 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2715                         enum flag_idn idn, bool *flag_res)
2716 {
2717         struct ufs_query_req *request = NULL;
2718         struct ufs_query_res *response = NULL;
2719         int err, index = 0, selector = 0;
2720         int timeout = QUERY_REQ_TIMEOUT;
2721
2722         BUG_ON(!hba);
2723
2724         ufshcd_hold(hba, false);
2725         mutex_lock(&hba->dev_cmd.lock);
2726         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2727                         selector);
2728
2729         switch (opcode) {
2730         case UPIU_QUERY_OPCODE_SET_FLAG:
2731         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2732         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2733                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2734                 break;
2735         case UPIU_QUERY_OPCODE_READ_FLAG:
2736                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2737                 if (!flag_res) {
2738                         /* No dummy reads */
2739                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
2740                                         __func__);
2741                         err = -EINVAL;
2742                         goto out_unlock;
2743                 }
2744                 break;
2745         default:
2746                 dev_err(hba->dev,
2747                         "%s: Expected query flag opcode but got = %d\n",
2748                         __func__, opcode);
2749                 err = -EINVAL;
2750                 goto out_unlock;
2751         }
2752
2753         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2754
2755         if (err) {
2756                 dev_err(hba->dev,
2757                         "%s: Sending flag query for idn %d failed, err = %d\n",
2758                         __func__, idn, err);
2759                 goto out_unlock;
2760         }
2761
2762         if (flag_res)
2763                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2764                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2765
2766 out_unlock:
2767         mutex_unlock(&hba->dev_cmd.lock);
2768         ufshcd_release(hba);
2769         return err;
2770 }
2771
2772 /**
2773  * ufshcd_query_attr - API function for sending attribute requests
2774  * @hba: per-adapter instance
2775  * @opcode: attribute opcode
2776  * @idn: attribute idn to access
2777  * @index: index field
2778  * @selector: selector field
2779  * @attr_val: the attribute value after the query request completes
2780  *
2781  * Returns 0 for success, non-zero in case of failure
2782 */
2783 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2784                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2785 {
2786         struct ufs_query_req *request = NULL;
2787         struct ufs_query_res *response = NULL;
2788         int err;
2789
2790         BUG_ON(!hba);
2791
2792         ufshcd_hold(hba, false);
2793         if (!attr_val) {
2794                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2795                                 __func__, opcode);
2796                 err = -EINVAL;
2797                 goto out;
2798         }
2799
2800         mutex_lock(&hba->dev_cmd.lock);
2801         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2802                         selector);
2803
2804         switch (opcode) {
2805         case UPIU_QUERY_OPCODE_WRITE_ATTR:
2806                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2807                 request->upiu_req.value = cpu_to_be32(*attr_val);
2808                 break;
2809         case UPIU_QUERY_OPCODE_READ_ATTR:
2810                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2811                 break;
2812         default:
2813                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2814                                 __func__, opcode);
2815                 err = -EINVAL;
2816                 goto out_unlock;
2817         }
2818
2819         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2820
2821         if (err) {
2822                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2823                                 __func__, opcode, idn, index, err);
2824                 goto out_unlock;
2825         }
2826
2827         *attr_val = be32_to_cpu(response->upiu_res.value);
2828
2829 out_unlock:
2830         mutex_unlock(&hba->dev_cmd.lock);
2831 out:
2832         ufshcd_release(hba);
2833         return err;
2834 }
2835
2836 /**
2837  * ufshcd_query_attr_retry() - API function for sending query
2838  * attribute with retries
2839  * @hba: per-adapter instance
2840  * @opcode: attribute opcode
2841  * @idn: attribute idn to access
2842  * @index: index field
2843  * @selector: selector field
2844  * @attr_val: the attribute value after the query request
2845  * completes
2846  *
2847  * Returns 0 for success, non-zero in case of failure
2848 */
2849 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2850         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2851         u32 *attr_val)
2852 {
2853         int ret = 0;
2854         u32 retries;
2855
2856         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2857                 ret = ufshcd_query_attr(hba, opcode, idn, index,
2858                                                 selector, attr_val);
2859                 if (ret)
2860                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2861                                 __func__, ret, retries);
2862                 else
2863                         break;
2864         }
2865
2866         if (ret)
2867                 dev_err(hba->dev,
2868                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2869                         __func__, idn, ret, QUERY_REQ_RETRIES);
2870         return ret;
2871 }
2872
2873 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
2874                         enum query_opcode opcode, enum desc_idn idn, u8 index,
2875                         u8 selector, u8 *desc_buf, int *buf_len)
2876 {
2877         struct ufs_query_req *request = NULL;
2878         struct ufs_query_res *response = NULL;
2879         int err;
2880
2881         BUG_ON(!hba);
2882
2883         ufshcd_hold(hba, false);
2884         if (!desc_buf) {
2885                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2886                                 __func__, opcode);
2887                 err = -EINVAL;
2888                 goto out;
2889         }
2890
2891         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
2892                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2893                                 __func__, *buf_len);
2894                 err = -EINVAL;
2895                 goto out;
2896         }
2897
2898         mutex_lock(&hba->dev_cmd.lock);
2899         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2900                         selector);
2901         hba->dev_cmd.query.descriptor = desc_buf;
2902         request->upiu_req.length = cpu_to_be16(*buf_len);
2903
2904         switch (opcode) {
2905         case UPIU_QUERY_OPCODE_WRITE_DESC:
2906                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2907                 break;
2908         case UPIU_QUERY_OPCODE_READ_DESC:
2909                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2910                 break;
2911         default:
2912                 dev_err(hba->dev,
2913                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2914                                 __func__, opcode);
2915                 err = -EINVAL;
2916                 goto out_unlock;
2917         }
2918
2919         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2920
2921         if (err) {
2922                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2923                                 __func__, opcode, idn, index, err);
2924                 goto out_unlock;
2925         }
2926
2927         *buf_len = be16_to_cpu(response->upiu_res.length);
2928
2929 out_unlock:
2930         hba->dev_cmd.query.descriptor = NULL;
2931         mutex_unlock(&hba->dev_cmd.lock);
2932 out:
2933         ufshcd_release(hba);
2934         return err;
2935 }
2936
2937 /**
2938  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2939  * @hba: per-adapter instance
2940  * @opcode: attribute opcode
2941  * @idn: attribute idn to access
2942  * @index: index field
2943  * @selector: selector field
2944  * @desc_buf: the buffer that contains the descriptor
2945  * @buf_len: length parameter passed to the device
2946  *
2947  * Returns 0 for success, non-zero in case of failure.
2948  * The buf_len parameter will contain, on return, the length parameter
2949  * received on the response.
2950  */
2951 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
2952                                   enum query_opcode opcode,
2953                                   enum desc_idn idn, u8 index,
2954                                   u8 selector,
2955                                   u8 *desc_buf, int *buf_len)
2956 {
2957         int err;
2958         int retries;
2959
2960         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2961                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
2962                                                 selector, desc_buf, buf_len);
2963                 if (!err || err == -EINVAL)
2964                         break;
2965         }
2966
2967         return err;
2968 }
2969
2970 /**
2971  * ufshcd_read_desc_length - read the specified descriptor length from header
2972  * @hba: Pointer to adapter instance
2973  * @desc_id: descriptor idn value
2974  * @desc_index: descriptor index
2975  * @desc_length: pointer to variable to read the length of descriptor
2976  *
2977  * Return 0 in case of success, non-zero otherwise
2978  */
2979 static int ufshcd_read_desc_length(struct ufs_hba *hba,
2980         enum desc_idn desc_id,
2981         int desc_index,
2982         int *desc_length)
2983 {
2984         int ret;
2985         u8 header[QUERY_DESC_HDR_SIZE];
2986         int header_len = QUERY_DESC_HDR_SIZE;
2987
2988         if (desc_id >= QUERY_DESC_IDN_MAX)
2989                 return -EINVAL;
2990
2991         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2992                                         desc_id, desc_index, 0, header,
2993                                         &header_len);
2994
2995         if (ret) {
2996                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
2997                         __func__, desc_id);
2998                 return ret;
2999         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3000                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3001                         __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3002                         desc_id);
3003                 ret = -EINVAL;
3004         }
3005
3006         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3007         return ret;
3008
3009 }
3010
3011 /**
3012  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3013  * @hba: Pointer to adapter instance
3014  * @desc_id: descriptor idn value
3015  * @desc_len: mapped desc length (out)
3016  *
3017  * Return 0 in case of success, non-zero otherwise
3018  */
3019 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3020         enum desc_idn desc_id, int *desc_len)
3021 {
3022         switch (desc_id) {
3023         case QUERY_DESC_IDN_DEVICE:
3024                 *desc_len = hba->desc_size.dev_desc;
3025                 break;
3026         case QUERY_DESC_IDN_POWER:
3027                 *desc_len = hba->desc_size.pwr_desc;
3028                 break;
3029         case QUERY_DESC_IDN_GEOMETRY:
3030                 *desc_len = hba->desc_size.geom_desc;
3031                 break;
3032         case QUERY_DESC_IDN_CONFIGURATION:
3033                 *desc_len = hba->desc_size.conf_desc;
3034                 break;
3035         case QUERY_DESC_IDN_UNIT:
3036                 *desc_len = hba->desc_size.unit_desc;
3037                 break;
3038         case QUERY_DESC_IDN_INTERCONNECT:
3039                 *desc_len = hba->desc_size.interc_desc;
3040                 break;
3041         case QUERY_DESC_IDN_STRING:
3042                 *desc_len = QUERY_DESC_MAX_SIZE;
3043                 break;
3044         case QUERY_DESC_IDN_HEALTH:
3045                 *desc_len = hba->desc_size.hlth_desc;
3046                 break;
3047         case QUERY_DESC_IDN_RFU_0:
3048         case QUERY_DESC_IDN_RFU_1:
3049                 *desc_len = 0;
3050                 break;
3051         default:
3052                 *desc_len = 0;
3053                 return -EINVAL;
3054         }
3055         return 0;
3056 }
3057 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3058
3059 /**
3060  * ufshcd_read_desc_param - read the specified descriptor parameter
3061  * @hba: Pointer to adapter instance
3062  * @desc_id: descriptor idn value
3063  * @desc_index: descriptor index
3064  * @param_offset: offset of the parameter to read
3065  * @param_read_buf: pointer to buffer where parameter would be read
3066  * @param_size: sizeof(param_read_buf)
3067  *
3068  * Return 0 in case of success, non-zero otherwise
3069  */
3070 int ufshcd_read_desc_param(struct ufs_hba *hba,
3071                            enum desc_idn desc_id,
3072                            int desc_index,
3073                            u8 param_offset,
3074                            u8 *param_read_buf,
3075                            u8 param_size)
3076 {
3077         int ret;
3078         u8 *desc_buf;
3079         int buff_len;
3080         bool is_kmalloc = true;
3081
3082         /* Safety check */
3083         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3084                 return -EINVAL;
3085
3086         /* Get the max length of descriptor from structure filled up at probe
3087          * time.
3088          */
3089         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3090
3091         /* Sanity checks */
3092         if (ret || !buff_len) {
3093                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3094                         __func__);
3095                 return ret;
3096         }
3097
3098         /* Check whether we need temp memory */
3099         if (param_offset != 0 || param_size < buff_len) {
3100                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3101                 if (!desc_buf)
3102                         return -ENOMEM;
3103         } else {
3104                 desc_buf = param_read_buf;
3105                 is_kmalloc = false;
3106         }
3107
3108         /* Request for full descriptor */
3109         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3110                                         desc_id, desc_index, 0,
3111                                         desc_buf, &buff_len);
3112
3113         if (ret) {
3114                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3115                         __func__, desc_id, desc_index, param_offset, ret);
3116                 goto out;
3117         }
3118
3119         /* Sanity check */
3120         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3121                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3122                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3123                 ret = -EINVAL;
3124                 goto out;
3125         }
3126
3127         /* Check wherher we will not copy more data, than available */
3128         if (is_kmalloc && param_size > buff_len)
3129                 param_size = buff_len;
3130
3131         if (is_kmalloc)
3132                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3133 out:
3134         if (is_kmalloc)
3135                 kfree(desc_buf);
3136         return ret;
3137 }
3138
3139 static inline int ufshcd_read_desc(struct ufs_hba *hba,
3140                                    enum desc_idn desc_id,
3141                                    int desc_index,
3142                                    void *buf,
3143                                    u32 size)
3144 {
3145         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3146 }
3147
3148
3149 /**
3150  * struct uc_string_id - unicode string
3151  *
3152  * @len: size of this descriptor inclusive
3153  * @type: descriptor type
3154  * @uc: unicode string character
3155  */
3156 struct uc_string_id {
3157         u8 len;
3158         u8 type;
3159         wchar_t uc[0];
3160 } __packed;
3161
3162 /* replace non-printable or non-ASCII characters with spaces */
3163 static inline char ufshcd_remove_non_printable(u8 ch)
3164 {
3165         return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3166 }
3167
3168 /**
3169  * ufshcd_read_string_desc - read string descriptor
3170  * @hba: pointer to adapter instance
3171  * @desc_index: descriptor index
3172  * @buf: pointer to buffer where descriptor would be read,
3173  *       the caller should free the memory.
3174  * @ascii: if true convert from unicode to ascii characters
3175  *         null terminated string.
3176  *
3177  * Return:
3178  * *      string size on success.
3179  * *      -ENOMEM: on allocation failure
3180  * *      -EINVAL: on a wrong parameter
3181  */
3182 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3183                             u8 **buf, bool ascii)
3184 {
3185         struct uc_string_id *uc_str;
3186         u8 *str;
3187         int ret;
3188
3189         if (!buf)
3190                 return -EINVAL;
3191
3192         uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3193         if (!uc_str)
3194                 return -ENOMEM;
3195
3196         ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING,
3197                                desc_index, uc_str,
3198                                QUERY_DESC_MAX_SIZE);
3199         if (ret < 0) {
3200                 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3201                         QUERY_REQ_RETRIES, ret);
3202                 str = NULL;
3203                 goto out;
3204         }
3205
3206         if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3207                 dev_dbg(hba->dev, "String Desc is of zero length\n");
3208                 str = NULL;
3209                 ret = 0;
3210                 goto out;
3211         }
3212
3213         if (ascii) {
3214                 ssize_t ascii_len;
3215                 int i;
3216                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3217                 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3218                 str = kzalloc(ascii_len, GFP_KERNEL);
3219                 if (!str) {
3220                         ret = -ENOMEM;
3221                         goto out;
3222                 }
3223
3224                 /*
3225                  * the descriptor contains string in UTF16 format
3226                  * we need to convert to utf-8 so it can be displayed
3227                  */
3228                 ret = utf16s_to_utf8s(uc_str->uc,
3229                                       uc_str->len - QUERY_DESC_HDR_SIZE,
3230                                       UTF16_BIG_ENDIAN, str, ascii_len);
3231
3232                 /* replace non-printable or non-ASCII characters with spaces */
3233                 for (i = 0; i < ret; i++)
3234                         str[i] = ufshcd_remove_non_printable(str[i]);
3235
3236                 str[ret++] = '\0';
3237
3238         } else {
3239                 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3240                 if (!str) {
3241                         ret = -ENOMEM;
3242                         goto out;
3243                 }
3244                 ret = uc_str->len;
3245         }
3246 out:
3247         *buf = str;
3248         kfree(uc_str);
3249         return ret;
3250 }
3251
3252 /**
3253  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3254  * @hba: Pointer to adapter instance
3255  * @lun: lun id
3256  * @param_offset: offset of the parameter to read
3257  * @param_read_buf: pointer to buffer where parameter would be read
3258  * @param_size: sizeof(param_read_buf)
3259  *
3260  * Return 0 in case of success, non-zero otherwise
3261  */
3262 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3263                                               int lun,
3264                                               enum unit_desc_param param_offset,
3265                                               u8 *param_read_buf,
3266                                               u32 param_size)
3267 {
3268         /*
3269          * Unit descriptors are only available for general purpose LUs (LUN id
3270          * from 0 to 7) and RPMB Well known LU.
3271          */
3272         if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3273                 return -EOPNOTSUPP;
3274
3275         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3276                                       param_offset, param_read_buf, param_size);
3277 }
3278
3279 /**
3280  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3281  * @hba: per adapter instance
3282  *
3283  * 1. Allocate DMA memory for Command Descriptor array
3284  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3285  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3286  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3287  *      (UTMRDL)
3288  * 4. Allocate memory for local reference block(lrb).
3289  *
3290  * Returns 0 for success, non-zero in case of failure
3291  */
3292 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3293 {
3294         size_t utmrdl_size, utrdl_size, ucdl_size;
3295
3296         /* Allocate memory for UTP command descriptors */
3297         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3298         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3299                                                   ucdl_size,
3300                                                   &hba->ucdl_dma_addr,
3301                                                   GFP_KERNEL);
3302
3303         /*
3304          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3305          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3306          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3307          * be aligned to 128 bytes as well
3308          */
3309         if (!hba->ucdl_base_addr ||
3310             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3311                 dev_err(hba->dev,
3312                         "Command Descriptor Memory allocation failed\n");
3313                 goto out;
3314         }
3315
3316         /*
3317          * Allocate memory for UTP Transfer descriptors
3318          * UFSHCI requires 1024 byte alignment of UTRD
3319          */
3320         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3321         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3322                                                    utrdl_size,
3323                                                    &hba->utrdl_dma_addr,
3324                                                    GFP_KERNEL);
3325         if (!hba->utrdl_base_addr ||
3326             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3327                 dev_err(hba->dev,
3328                         "Transfer Descriptor Memory allocation failed\n");
3329                 goto out;
3330         }
3331
3332         /*
3333          * Allocate memory for UTP Task Management descriptors
3334          * UFSHCI requires 1024 byte alignment of UTMRD
3335          */
3336         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3337         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3338                                                     utmrdl_size,
3339                                                     &hba->utmrdl_dma_addr,
3340                                                     GFP_KERNEL);
3341         if (!hba->utmrdl_base_addr ||
3342             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3343                 dev_err(hba->dev,
3344                 "Task Management Descriptor Memory allocation failed\n");
3345                 goto out;
3346         }
3347
3348         /* Allocate memory for local reference block */
3349         hba->lrb = devm_kcalloc(hba->dev,
3350                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3351                                 GFP_KERNEL);
3352         if (!hba->lrb) {
3353                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3354                 goto out;
3355         }
3356         return 0;
3357 out:
3358         return -ENOMEM;
3359 }
3360
3361 /**
3362  * ufshcd_host_memory_configure - configure local reference block with
3363  *                              memory offsets
3364  * @hba: per adapter instance
3365  *
3366  * Configure Host memory space
3367  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3368  * address.
3369  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3370  * and PRDT offset.
3371  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3372  * into local reference block.
3373  */
3374 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3375 {
3376         struct utp_transfer_cmd_desc *cmd_descp;
3377         struct utp_transfer_req_desc *utrdlp;
3378         dma_addr_t cmd_desc_dma_addr;
3379         dma_addr_t cmd_desc_element_addr;
3380         u16 response_offset;
3381         u16 prdt_offset;
3382         int cmd_desc_size;
3383         int i;
3384
3385         utrdlp = hba->utrdl_base_addr;
3386         cmd_descp = hba->ucdl_base_addr;
3387
3388         response_offset =
3389                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3390         prdt_offset =
3391                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3392
3393         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3394         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3395
3396         for (i = 0; i < hba->nutrs; i++) {
3397                 /* Configure UTRD with command descriptor base address */
3398                 cmd_desc_element_addr =
3399                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3400                 utrdlp[i].command_desc_base_addr_lo =
3401                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3402                 utrdlp[i].command_desc_base_addr_hi =
3403                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3404
3405                 /* Response upiu and prdt offset should be in double words */
3406                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3407                         utrdlp[i].response_upiu_offset =
3408                                 cpu_to_le16(response_offset);
3409                         utrdlp[i].prd_table_offset =
3410                                 cpu_to_le16(prdt_offset);
3411                         utrdlp[i].response_upiu_length =
3412                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3413                 } else {
3414                         utrdlp[i].response_upiu_offset =
3415                                 cpu_to_le16((response_offset >> 2));
3416                         utrdlp[i].prd_table_offset =
3417                                 cpu_to_le16((prdt_offset >> 2));
3418                         utrdlp[i].response_upiu_length =
3419                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3420                 }
3421
3422                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
3423                 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3424                                 (i * sizeof(struct utp_transfer_req_desc));
3425                 hba->lrb[i].ucd_req_ptr =
3426                         (struct utp_upiu_req *)(cmd_descp + i);
3427                 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
3428                 hba->lrb[i].ucd_rsp_ptr =
3429                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
3430                 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3431                                 response_offset;
3432                 hba->lrb[i].ucd_prdt_ptr =
3433                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
3434                 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3435                                 prdt_offset;
3436         }
3437 }
3438
3439 /**
3440  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3441  * @hba: per adapter instance
3442  *
3443  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3444  * in order to initialize the Unipro link startup procedure.
3445  * Once the Unipro links are up, the device connected to the controller
3446  * is detected.
3447  *
3448  * Returns 0 on success, non-zero value on failure
3449  */
3450 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3451 {
3452         struct uic_command uic_cmd = {0};
3453         int ret;
3454
3455         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3456
3457         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3458         if (ret)
3459                 dev_dbg(hba->dev,
3460                         "dme-link-startup: error code %d\n", ret);
3461         return ret;
3462 }
3463 /**
3464  * ufshcd_dme_reset - UIC command for DME_RESET
3465  * @hba: per adapter instance
3466  *
3467  * DME_RESET command is issued in order to reset UniPro stack.
3468  * This function now deal with cold reset.
3469  *
3470  * Returns 0 on success, non-zero value on failure
3471  */
3472 static int ufshcd_dme_reset(struct ufs_hba *hba)
3473 {
3474         struct uic_command uic_cmd = {0};
3475         int ret;
3476
3477         uic_cmd.command = UIC_CMD_DME_RESET;
3478
3479         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3480         if (ret)
3481                 dev_err(hba->dev,
3482                         "dme-reset: error code %d\n", ret);
3483
3484         return ret;
3485 }
3486
3487 /**
3488  * ufshcd_dme_enable - UIC command for DME_ENABLE
3489  * @hba: per adapter instance
3490  *
3491  * DME_ENABLE command is issued in order to enable UniPro stack.
3492  *
3493  * Returns 0 on success, non-zero value on failure
3494  */
3495 static int ufshcd_dme_enable(struct ufs_hba *hba)
3496 {
3497         struct uic_command uic_cmd = {0};
3498         int ret;
3499
3500         uic_cmd.command = UIC_CMD_DME_ENABLE;
3501
3502         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3503         if (ret)
3504                 dev_err(hba->dev,
3505                         "dme-reset: error code %d\n", ret);
3506
3507         return ret;
3508 }
3509
3510 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3511 {
3512         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3513         unsigned long min_sleep_time_us;
3514
3515         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3516                 return;
3517
3518         /*
3519          * last_dme_cmd_tstamp will be 0 only for 1st call to
3520          * this function
3521          */
3522         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3523                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3524         } else {
3525                 unsigned long delta =
3526                         (unsigned long) ktime_to_us(
3527                                 ktime_sub(ktime_get(),
3528                                 hba->last_dme_cmd_tstamp));
3529
3530                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3531                         min_sleep_time_us =
3532                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3533                 else
3534                         return; /* no more delay required */
3535         }
3536
3537         /* allow sleep for extra 50us if needed */
3538         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3539 }
3540
3541 /**
3542  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3543  * @hba: per adapter instance
3544  * @attr_sel: uic command argument1
3545  * @attr_set: attribute set type as uic command argument2
3546  * @mib_val: setting value as uic command argument3
3547  * @peer: indicate whether peer or local
3548  *
3549  * Returns 0 on success, non-zero value on failure
3550  */
3551 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3552                         u8 attr_set, u32 mib_val, u8 peer)
3553 {
3554         struct uic_command uic_cmd = {0};
3555         static const char *const action[] = {
3556                 "dme-set",
3557                 "dme-peer-set"
3558         };
3559         const char *set = action[!!peer];
3560         int ret;
3561         int retries = UFS_UIC_COMMAND_RETRIES;
3562
3563         uic_cmd.command = peer ?
3564                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3565         uic_cmd.argument1 = attr_sel;
3566         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3567         uic_cmd.argument3 = mib_val;
3568
3569         do {
3570                 /* for peer attributes we retry upon failure */
3571                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3572                 if (ret)
3573                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3574                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3575         } while (ret && peer && --retries);
3576
3577         if (ret)
3578                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3579                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3580                         UFS_UIC_COMMAND_RETRIES - retries);
3581
3582         return ret;
3583 }
3584 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3585
3586 /**
3587  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3588  * @hba: per adapter instance
3589  * @attr_sel: uic command argument1
3590  * @mib_val: the value of the attribute as returned by the UIC command
3591  * @peer: indicate whether peer or local
3592  *
3593  * Returns 0 on success, non-zero value on failure
3594  */
3595 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3596                         u32 *mib_val, u8 peer)
3597 {
3598         struct uic_command uic_cmd = {0};
3599         static const char *const action[] = {
3600                 "dme-get",
3601                 "dme-peer-get"
3602         };
3603         const char *get = action[!!peer];
3604         int ret;
3605         int retries = UFS_UIC_COMMAND_RETRIES;
3606         struct ufs_pa_layer_attr orig_pwr_info;
3607         struct ufs_pa_layer_attr temp_pwr_info;
3608         bool pwr_mode_change = false;
3609
3610         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3611                 orig_pwr_info = hba->pwr_info;
3612                 temp_pwr_info = orig_pwr_info;
3613
3614                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3615                     orig_pwr_info.pwr_rx == FAST_MODE) {
3616                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3617                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3618                         pwr_mode_change = true;
3619                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3620                     orig_pwr_info.pwr_rx == SLOW_MODE) {
3621                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3622                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3623                         pwr_mode_change = true;
3624                 }
3625                 if (pwr_mode_change) {
3626                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3627                         if (ret)
3628                                 goto out;
3629                 }
3630         }
3631
3632         uic_cmd.command = peer ?
3633                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3634         uic_cmd.argument1 = attr_sel;
3635
3636         do {
3637                 /* for peer attributes we retry upon failure */
3638                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3639                 if (ret)
3640                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3641                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
3642         } while (ret && peer && --retries);
3643
3644         if (ret)
3645                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3646                         get, UIC_GET_ATTR_ID(attr_sel),
3647                         UFS_UIC_COMMAND_RETRIES - retries);
3648
3649         if (mib_val && !ret)
3650                 *mib_val = uic_cmd.argument3;
3651
3652         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3653             && pwr_mode_change)
3654                 ufshcd_change_power_mode(hba, &orig_pwr_info);
3655 out:
3656         return ret;
3657 }
3658 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3659
3660 /**
3661  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3662  * state) and waits for it to take effect.
3663  *
3664  * @hba: per adapter instance
3665  * @cmd: UIC command to execute
3666  *
3667  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3668  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3669  * and device UniPro link and hence it's final completion would be indicated by
3670  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3671  * addition to normal UIC command completion Status (UCCS). This function only
3672  * returns after the relevant status bits indicate the completion.
3673  *
3674  * Returns 0 on success, non-zero value on failure
3675  */
3676 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3677 {
3678         struct completion uic_async_done;
3679         unsigned long flags;
3680         u8 status;
3681         int ret;
3682         bool reenable_intr = false;
3683
3684         mutex_lock(&hba->uic_cmd_mutex);
3685         init_completion(&uic_async_done);
3686         ufshcd_add_delay_before_dme_cmd(hba);
3687
3688         spin_lock_irqsave(hba->host->host_lock, flags);
3689         hba->uic_async_done = &uic_async_done;
3690         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3691                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3692                 /*
3693                  * Make sure UIC command completion interrupt is disabled before
3694                  * issuing UIC command.
3695                  */
3696                 wmb();
3697                 reenable_intr = true;
3698         }
3699         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3700         spin_unlock_irqrestore(hba->host->host_lock, flags);
3701         if (ret) {
3702                 dev_err(hba->dev,
3703                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3704                         cmd->command, cmd->argument3, ret);
3705                 goto out;
3706         }
3707
3708         if (!wait_for_completion_timeout(hba->uic_async_done,
3709                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3710                 dev_err(hba->dev,
3711                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3712                         cmd->command, cmd->argument3);
3713                 ret = -ETIMEDOUT;
3714                 goto out;
3715         }
3716
3717         status = ufshcd_get_upmcrs(hba);
3718         if (status != PWR_LOCAL) {
3719                 dev_err(hba->dev,
3720                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
3721                         cmd->command, status);
3722                 ret = (status != PWR_OK) ? status : -1;
3723         }
3724 out:
3725         if (ret) {
3726                 ufshcd_print_host_state(hba);
3727                 ufshcd_print_pwr_info(hba);
3728                 ufshcd_print_host_regs(hba);
3729         }
3730
3731         spin_lock_irqsave(hba->host->host_lock, flags);
3732         hba->active_uic_cmd = NULL;
3733         hba->uic_async_done = NULL;
3734         if (reenable_intr)
3735                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
3736         spin_unlock_irqrestore(hba->host->host_lock, flags);
3737         mutex_unlock(&hba->uic_cmd_mutex);
3738
3739         return ret;
3740 }
3741
3742 /**
3743  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3744  *                              using DME_SET primitives.
3745  * @hba: per adapter instance
3746  * @mode: powr mode value
3747  *
3748  * Returns 0 on success, non-zero value on failure
3749  */
3750 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3751 {
3752         struct uic_command uic_cmd = {0};
3753         int ret;
3754
3755         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3756                 ret = ufshcd_dme_set(hba,
3757                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3758                 if (ret) {
3759                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3760                                                 __func__, ret);
3761                         goto out;
3762                 }
3763         }
3764
3765         uic_cmd.command = UIC_CMD_DME_SET;
3766         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3767         uic_cmd.argument3 = mode;
3768         ufshcd_hold(hba, false);
3769         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3770         ufshcd_release(hba);
3771
3772 out:
3773         return ret;
3774 }
3775
3776 static int ufshcd_link_recovery(struct ufs_hba *hba)
3777 {
3778         int ret;
3779         unsigned long flags;
3780
3781         spin_lock_irqsave(hba->host->host_lock, flags);
3782         hba->ufshcd_state = UFSHCD_STATE_RESET;
3783         ufshcd_set_eh_in_progress(hba);
3784         spin_unlock_irqrestore(hba->host->host_lock, flags);
3785
3786         /* Reset the attached device */
3787         ufshcd_vops_device_reset(hba);
3788
3789         ret = ufshcd_host_reset_and_restore(hba);
3790
3791         spin_lock_irqsave(hba->host->host_lock, flags);
3792         if (ret)
3793                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3794         ufshcd_clear_eh_in_progress(hba);
3795         spin_unlock_irqrestore(hba->host->host_lock, flags);
3796
3797         if (ret)
3798                 dev_err(hba->dev, "%s: link recovery failed, err %d",
3799                         __func__, ret);
3800
3801         return ret;
3802 }
3803
3804 static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3805 {
3806         int ret;
3807         struct uic_command uic_cmd = {0};
3808         ktime_t start = ktime_get();
3809
3810         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3811
3812         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
3813         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3814         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3815                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3816
3817         if (ret) {
3818                 int err;
3819
3820                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3821                         __func__, ret);
3822
3823                 /*
3824                  * If link recovery fails then return error code returned from
3825                  * ufshcd_link_recovery().
3826                  * If link recovery succeeds then return -EAGAIN to attempt
3827                  * hibern8 enter retry again.
3828                  */
3829                 err = ufshcd_link_recovery(hba);
3830                 if (err) {
3831                         dev_err(hba->dev, "%s: link recovery failed", __func__);
3832                         ret = err;
3833                 } else {
3834                         ret = -EAGAIN;
3835                 }
3836         } else
3837                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3838                                                                 POST_CHANGE);
3839
3840         return ret;
3841 }
3842
3843 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3844 {
3845         int ret = 0, retries;
3846
3847         for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3848                 ret = __ufshcd_uic_hibern8_enter(hba);
3849                 if (!ret)
3850                         goto out;
3851         }
3852 out:
3853         return ret;
3854 }
3855
3856 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3857 {
3858         struct uic_command uic_cmd = {0};
3859         int ret;
3860         ktime_t start = ktime_get();
3861
3862         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3863
3864         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3865         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3866         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3867                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3868
3869         if (ret) {
3870                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3871                         __func__, ret);
3872                 ret = ufshcd_link_recovery(hba);
3873         } else {
3874                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3875                                                                 POST_CHANGE);
3876                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3877                 hba->ufs_stats.hibern8_exit_cnt++;
3878         }
3879
3880         return ret;
3881 }
3882 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
3883
3884 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
3885 {
3886         unsigned long flags;
3887
3888         if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT))
3889                 return;
3890
3891         spin_lock_irqsave(hba->host->host_lock, flags);
3892         if (hba->ahit == ahit)
3893                 goto out_unlock;
3894         hba->ahit = ahit;
3895         if (!pm_runtime_suspended(hba->dev))
3896                 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3897 out_unlock:
3898         spin_unlock_irqrestore(hba->host->host_lock, flags);
3899 }
3900 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
3901
3902 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
3903 {
3904         unsigned long flags;
3905
3906         if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
3907                 return;
3908
3909         spin_lock_irqsave(hba->host->host_lock, flags);
3910         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3911         spin_unlock_irqrestore(hba->host->host_lock, flags);
3912 }
3913
3914  /**
3915  * ufshcd_init_pwr_info - setting the POR (power on reset)
3916  * values in hba power info
3917  * @hba: per-adapter instance
3918  */
3919 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3920 {
3921         hba->pwr_info.gear_rx = UFS_PWM_G1;
3922         hba->pwr_info.gear_tx = UFS_PWM_G1;
3923         hba->pwr_info.lane_rx = 1;
3924         hba->pwr_info.lane_tx = 1;
3925         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3926         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3927         hba->pwr_info.hs_rate = 0;
3928 }
3929
3930 /**
3931  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3932  * @hba: per-adapter instance
3933  */
3934 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
3935 {
3936         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
3937
3938         if (hba->max_pwr_info.is_valid)
3939                 return 0;
3940
3941         pwr_info->pwr_tx = FAST_MODE;
3942         pwr_info->pwr_rx = FAST_MODE;
3943         pwr_info->hs_rate = PA_HS_MODE_B;
3944
3945         /* Get the connected lane count */
3946         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
3947                         &pwr_info->lane_rx);
3948         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3949                         &pwr_info->lane_tx);
3950
3951         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
3952                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
3953                                 __func__,
3954                                 pwr_info->lane_rx,
3955                                 pwr_info->lane_tx);
3956                 return -EINVAL;
3957         }
3958
3959         /*
3960          * First, get the maximum gears of HS speed.
3961          * If a zero value, it means there is no HSGEAR capability.
3962          * Then, get the maximum gears of PWM speed.
3963          */
3964         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
3965         if (!pwr_info->gear_rx) {
3966                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3967                                 &pwr_info->gear_rx);
3968                 if (!pwr_info->gear_rx) {
3969                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
3970                                 __func__, pwr_info->gear_rx);
3971                         return -EINVAL;
3972                 }
3973                 pwr_info->pwr_rx = SLOW_MODE;
3974         }
3975
3976         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
3977                         &pwr_info->gear_tx);
3978         if (!pwr_info->gear_tx) {
3979                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
3980                                 &pwr_info->gear_tx);
3981                 if (!pwr_info->gear_tx) {
3982                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
3983                                 __func__, pwr_info->gear_tx);
3984                         return -EINVAL;
3985                 }
3986                 pwr_info->pwr_tx = SLOW_MODE;
3987         }
3988
3989         hba->max_pwr_info.is_valid = true;
3990         return 0;
3991 }
3992
3993 static int ufshcd_change_power_mode(struct ufs_hba *hba,
3994                              struct ufs_pa_layer_attr *pwr_mode)
3995 {
3996         int ret;
3997
3998         /* if already configured to the requested pwr_mode */
3999         if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4000             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4001             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4002             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4003             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4004             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4005             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4006                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4007                 return 0;
4008         }
4009
4010         /*
4011          * Configure attributes for power mode change with below.
4012          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4013          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4014          * - PA_HSSERIES
4015          */
4016         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4017         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4018                         pwr_mode->lane_rx);
4019         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4020                         pwr_mode->pwr_rx == FAST_MODE)
4021                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4022         else
4023                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4024
4025         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4026         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4027                         pwr_mode->lane_tx);
4028         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4029                         pwr_mode->pwr_tx == FAST_MODE)
4030                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4031         else
4032                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4033
4034         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4035             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4036             pwr_mode->pwr_rx == FAST_MODE ||
4037             pwr_mode->pwr_tx == FAST_MODE)
4038                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4039                                                 pwr_mode->hs_rate);
4040
4041         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4042                         DL_FC0ProtectionTimeOutVal_Default);
4043         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4044                         DL_TC0ReplayTimeOutVal_Default);
4045         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4046                         DL_AFC0ReqTimeOutVal_Default);
4047         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4048                         DL_FC1ProtectionTimeOutVal_Default);
4049         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4050                         DL_TC1ReplayTimeOutVal_Default);
4051         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4052                         DL_AFC1ReqTimeOutVal_Default);
4053
4054         ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4055                         DL_FC0ProtectionTimeOutVal_Default);
4056         ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4057                         DL_TC0ReplayTimeOutVal_Default);
4058         ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4059                         DL_AFC0ReqTimeOutVal_Default);
4060
4061         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4062                         | pwr_mode->pwr_tx);
4063
4064         if (ret) {
4065                 dev_err(hba->dev,
4066                         "%s: power mode change failed %d\n", __func__, ret);
4067         } else {
4068                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4069                                                                 pwr_mode);
4070
4071                 memcpy(&hba->pwr_info, pwr_mode,
4072                         sizeof(struct ufs_pa_layer_attr));
4073         }
4074
4075         return ret;
4076 }
4077
4078 /**
4079  * ufshcd_config_pwr_mode - configure a new power mode
4080  * @hba: per-adapter instance
4081  * @desired_pwr_mode: desired power configuration
4082  */
4083 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4084                 struct ufs_pa_layer_attr *desired_pwr_mode)
4085 {
4086         struct ufs_pa_layer_attr final_params = { 0 };
4087         int ret;
4088
4089         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4090                                         desired_pwr_mode, &final_params);
4091
4092         if (ret)
4093                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4094
4095         ret = ufshcd_change_power_mode(hba, &final_params);
4096         if (!ret)
4097                 ufshcd_print_pwr_info(hba);
4098
4099         return ret;
4100 }
4101 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4102
4103 /**
4104  * ufshcd_complete_dev_init() - checks device readiness
4105  * @hba: per-adapter instance
4106  *
4107  * Set fDeviceInit flag and poll until device toggles it.
4108  */
4109 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4110 {
4111         int i;
4112         int err;
4113         bool flag_res = 1;
4114
4115         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4116                 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
4117         if (err) {
4118                 dev_err(hba->dev,
4119                         "%s setting fDeviceInit flag failed with error %d\n",
4120                         __func__, err);
4121                 goto out;
4122         }
4123
4124         /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4125         for (i = 0; i < 1000 && !err && flag_res; i++)
4126                 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4127                         QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4128
4129         if (err)
4130                 dev_err(hba->dev,
4131                         "%s reading fDeviceInit flag failed with error %d\n",
4132                         __func__, err);
4133         else if (flag_res)
4134                 dev_err(hba->dev,
4135                         "%s fDeviceInit was not cleared by the device\n",
4136                         __func__);
4137
4138 out:
4139         return err;
4140 }
4141
4142 /**
4143  * ufshcd_make_hba_operational - Make UFS controller operational
4144  * @hba: per adapter instance
4145  *
4146  * To bring UFS host controller to operational state,
4147  * 1. Enable required interrupts
4148  * 2. Configure interrupt aggregation
4149  * 3. Program UTRL and UTMRL base address
4150  * 4. Configure run-stop-registers
4151  *
4152  * Returns 0 on success, non-zero value on failure
4153  */
4154 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4155 {
4156         int err = 0;
4157         u32 reg;
4158
4159         /* Enable required interrupts */
4160         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4161
4162         /* Configure interrupt aggregation */
4163         if (ufshcd_is_intr_aggr_allowed(hba))
4164                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4165         else
4166                 ufshcd_disable_intr_aggr(hba);
4167
4168         /* Configure UTRL and UTMRL base address registers */
4169         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4170                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4171         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4172                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4173         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4174                         REG_UTP_TASK_REQ_LIST_BASE_L);
4175         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4176                         REG_UTP_TASK_REQ_LIST_BASE_H);
4177
4178         /*
4179          * Make sure base address and interrupt setup are updated before
4180          * enabling the run/stop registers below.
4181          */
4182         wmb();
4183
4184         /*
4185          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4186          */
4187         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4188         if (!(ufshcd_get_lists_status(reg))) {
4189                 ufshcd_enable_run_stop_reg(hba);
4190         } else {
4191                 dev_err(hba->dev,
4192                         "Host controller not ready to process requests");
4193                 err = -EIO;
4194                 goto out;
4195         }
4196
4197 out:
4198         return err;
4199 }
4200 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4201
4202 /**
4203  * ufshcd_hba_stop - Send controller to reset state
4204  * @hba: per adapter instance
4205  * @can_sleep: perform sleep or just spin
4206  */
4207 static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4208 {
4209         int err;
4210
4211         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4212         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4213                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4214                                         10, 1, can_sleep);
4215         if (err)
4216                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4217 }
4218
4219 /**
4220  * ufshcd_hba_execute_hce - initialize the controller
4221  * @hba: per adapter instance
4222  *
4223  * The controller resets itself and controller firmware initialization
4224  * sequence kicks off. When controller is ready it will set
4225  * the Host Controller Enable bit to 1.
4226  *
4227  * Returns 0 on success, non-zero value on failure
4228  */
4229 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4230 {
4231         int retry;
4232
4233         if (!ufshcd_is_hba_active(hba))
4234                 /* change controller state to "reset state" */
4235                 ufshcd_hba_stop(hba, true);
4236
4237         /* UniPro link is disabled at this point */
4238         ufshcd_set_link_off(hba);
4239
4240         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4241
4242         /* start controller initialization sequence */
4243         ufshcd_hba_start(hba);
4244
4245         /*
4246          * To initialize a UFS host controller HCE bit must be set to 1.
4247          * During initialization the HCE bit value changes from 1->0->1.
4248          * When the host controller completes initialization sequence
4249          * it sets the value of HCE bit to 1. The same HCE bit is read back
4250          * to check if the controller has completed initialization sequence.
4251          * So without this delay the value HCE = 1, set in the previous
4252          * instruction might be read back.
4253          * This delay can be changed based on the controller.
4254          */
4255         usleep_range(1000, 1100);
4256
4257         /* wait for the host controller to complete initialization */
4258         retry = 10;
4259         while (ufshcd_is_hba_active(hba)) {
4260                 if (retry) {
4261                         retry--;
4262                 } else {
4263                         dev_err(hba->dev,
4264                                 "Controller enable failed\n");
4265                         return -EIO;
4266                 }
4267                 usleep_range(5000, 5100);
4268         }
4269
4270         /* enable UIC related interrupts */
4271         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4272
4273         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4274
4275         return 0;
4276 }
4277
4278 int ufshcd_hba_enable(struct ufs_hba *hba)
4279 {
4280         int ret;
4281
4282         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4283                 ufshcd_set_link_off(hba);
4284                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4285
4286                 /* enable UIC related interrupts */
4287                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4288                 ret = ufshcd_dme_reset(hba);
4289                 if (!ret) {
4290                         ret = ufshcd_dme_enable(hba);
4291                         if (!ret)
4292                                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4293                         if (ret)
4294                                 dev_err(hba->dev,
4295                                         "Host controller enable failed with non-hce\n");
4296                 }
4297         } else {
4298                 ret = ufshcd_hba_execute_hce(hba);
4299         }
4300
4301         return ret;
4302 }
4303 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4304
4305 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4306 {
4307         int tx_lanes, i, err = 0;
4308
4309         if (!peer)
4310                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4311                                &tx_lanes);
4312         else
4313                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4314                                     &tx_lanes);
4315         for (i = 0; i < tx_lanes; i++) {
4316                 if (!peer)
4317                         err = ufshcd_dme_set(hba,
4318                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4319                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4320                                         0);
4321                 else
4322                         err = ufshcd_dme_peer_set(hba,
4323                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4324                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4325                                         0);
4326                 if (err) {
4327                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4328                                 __func__, peer, i, err);
4329                         break;
4330                 }
4331         }
4332
4333         return err;
4334 }
4335
4336 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4337 {
4338         return ufshcd_disable_tx_lcc(hba, true);
4339 }
4340
4341 void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4342                             u32 reg)
4343 {
4344         reg_hist->reg[reg_hist->pos] = reg;
4345         reg_hist->tstamp[reg_hist->pos] = ktime_get();
4346         reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4347 }
4348 EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist);
4349
4350 /**
4351  * ufshcd_link_startup - Initialize unipro link startup
4352  * @hba: per adapter instance
4353  *
4354  * Returns 0 for success, non-zero in case of failure
4355  */
4356 static int ufshcd_link_startup(struct ufs_hba *hba)
4357 {
4358         int ret;
4359         int retries = DME_LINKSTARTUP_RETRIES;
4360         bool link_startup_again = false;
4361
4362         /*
4363          * If UFS device isn't active then we will have to issue link startup
4364          * 2 times to make sure the device state move to active.
4365          */
4366         if (!ufshcd_is_ufs_dev_active(hba))
4367                 link_startup_again = true;
4368
4369 link_startup:
4370         do {
4371                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4372
4373                 ret = ufshcd_dme_link_startup(hba);
4374
4375                 /* check if device is detected by inter-connect layer */
4376                 if (!ret && !ufshcd_is_device_present(hba)) {
4377                         ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4378                                                0);
4379                         dev_err(hba->dev, "%s: Device not present\n", __func__);
4380                         ret = -ENXIO;
4381                         goto out;
4382                 }
4383
4384                 /*
4385                  * DME link lost indication is only received when link is up,
4386                  * but we can't be sure if the link is up until link startup
4387                  * succeeds. So reset the local Uni-Pro and try again.
4388                  */
4389                 if (ret && ufshcd_hba_enable(hba)) {
4390                         ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4391                                                (u32)ret);
4392                         goto out;
4393                 }
4394         } while (ret && retries--);
4395
4396         if (ret) {
4397                 /* failed to get the link up... retire */
4398                 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4399                                        (u32)ret);
4400                 goto out;
4401         }
4402
4403         if (link_startup_again) {
4404                 link_startup_again = false;
4405                 retries = DME_LINKSTARTUP_RETRIES;
4406                 goto link_startup;
4407         }
4408
4409         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4410         ufshcd_init_pwr_info(hba);
4411         ufshcd_print_pwr_info(hba);
4412
4413         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4414                 ret = ufshcd_disable_device_tx_lcc(hba);
4415                 if (ret)
4416                         goto out;
4417         }
4418
4419         /* Include any host controller configuration via UIC commands */
4420         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4421         if (ret)
4422                 goto out;
4423
4424         ret = ufshcd_make_hba_operational(hba);
4425 out:
4426         if (ret) {
4427                 dev_err(hba->dev, "link startup failed %d\n", ret);
4428                 ufshcd_print_host_state(hba);
4429                 ufshcd_print_pwr_info(hba);
4430                 ufshcd_print_host_regs(hba);
4431         }
4432         return ret;
4433 }
4434
4435 /**
4436  * ufshcd_verify_dev_init() - Verify device initialization
4437  * @hba: per-adapter instance
4438  *
4439  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4440  * device Transport Protocol (UTP) layer is ready after a reset.
4441  * If the UTP layer at the device side is not initialized, it may
4442  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4443  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4444  */
4445 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4446 {
4447         int err = 0;
4448         int retries;
4449
4450         ufshcd_hold(hba, false);
4451         mutex_lock(&hba->dev_cmd.lock);
4452         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4453                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4454                                                NOP_OUT_TIMEOUT);
4455
4456                 if (!err || err == -ETIMEDOUT)
4457                         break;
4458
4459                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4460         }
4461         mutex_unlock(&hba->dev_cmd.lock);
4462         ufshcd_release(hba);
4463
4464         if (err)
4465                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4466         return err;
4467 }
4468
4469 /**
4470  * ufshcd_set_queue_depth - set lun queue depth
4471  * @sdev: pointer to SCSI device
4472  *
4473  * Read bLUQueueDepth value and activate scsi tagged command
4474  * queueing. For WLUN, queue depth is set to 1. For best-effort
4475  * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4476  * value that host can queue.
4477  */
4478 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4479 {
4480         int ret = 0;
4481         u8 lun_qdepth;
4482         struct ufs_hba *hba;
4483
4484         hba = shost_priv(sdev->host);
4485
4486         lun_qdepth = hba->nutrs;
4487         ret = ufshcd_read_unit_desc_param(hba,
4488                                           ufshcd_scsi_to_upiu_lun(sdev->lun),
4489                                           UNIT_DESC_PARAM_LU_Q_DEPTH,
4490                                           &lun_qdepth,
4491                                           sizeof(lun_qdepth));
4492
4493         /* Some WLUN doesn't support unit descriptor */
4494         if (ret == -EOPNOTSUPP)
4495                 lun_qdepth = 1;
4496         else if (!lun_qdepth)
4497                 /* eventually, we can figure out the real queue depth */
4498                 lun_qdepth = hba->nutrs;
4499         else
4500                 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4501
4502         dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4503                         __func__, lun_qdepth);
4504         scsi_change_queue_depth(sdev, lun_qdepth);
4505 }
4506
4507 /*
4508  * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4509  * @hba: per-adapter instance
4510  * @lun: UFS device lun id
4511  * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4512  *
4513  * Returns 0 in case of success and b_lu_write_protect status would be returned
4514  * @b_lu_write_protect parameter.
4515  * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4516  * Returns -EINVAL in case of invalid parameters passed to this function.
4517  */
4518 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4519                             u8 lun,
4520                             u8 *b_lu_write_protect)
4521 {
4522         int ret;
4523
4524         if (!b_lu_write_protect)
4525                 ret = -EINVAL;
4526         /*
4527          * According to UFS device spec, RPMB LU can't be write
4528          * protected so skip reading bLUWriteProtect parameter for
4529          * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4530          */
4531         else if (lun >= hba->dev_info.max_lu_supported)
4532                 ret = -ENOTSUPP;
4533         else
4534                 ret = ufshcd_read_unit_desc_param(hba,
4535                                           lun,
4536                                           UNIT_DESC_PARAM_LU_WR_PROTECT,
4537                                           b_lu_write_protect,
4538                                           sizeof(*b_lu_write_protect));
4539         return ret;
4540 }
4541
4542 /**
4543  * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4544  * status
4545  * @hba: per-adapter instance
4546  * @sdev: pointer to SCSI device
4547  *
4548  */
4549 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4550                                                     struct scsi_device *sdev)
4551 {
4552         if (hba->dev_info.f_power_on_wp_en &&
4553             !hba->dev_info.is_lu_power_on_wp) {
4554                 u8 b_lu_write_protect;
4555
4556                 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4557                                       &b_lu_write_protect) &&
4558                     (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4559                         hba->dev_info.is_lu_power_on_wp = true;
4560         }
4561 }
4562
4563 /**
4564  * ufshcd_slave_alloc - handle initial SCSI device configurations
4565  * @sdev: pointer to SCSI device
4566  *
4567  * Returns success
4568  */
4569 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4570 {
4571         struct ufs_hba *hba;
4572
4573         hba = shost_priv(sdev->host);
4574
4575         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4576         sdev->use_10_for_ms = 1;
4577
4578         /* DBD field should be set to 1 in mode sense(10) */
4579         sdev->set_dbd_for_ms = 1;
4580
4581         /* allow SCSI layer to restart the device in case of errors */
4582         sdev->allow_restart = 1;
4583
4584         /* REPORT SUPPORTED OPERATION CODES is not supported */
4585         sdev->no_report_opcodes = 1;
4586
4587         /* WRITE_SAME command is not supported */
4588         sdev->no_write_same = 1;
4589
4590         ufshcd_set_queue_depth(sdev);
4591
4592         ufshcd_get_lu_power_on_wp_status(hba, sdev);
4593
4594         return 0;
4595 }
4596
4597 /**
4598  * ufshcd_change_queue_depth - change queue depth
4599  * @sdev: pointer to SCSI device
4600  * @depth: required depth to set
4601  *
4602  * Change queue depth and make sure the max. limits are not crossed.
4603  */
4604 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4605 {
4606         struct ufs_hba *hba = shost_priv(sdev->host);
4607
4608         if (depth > hba->nutrs)
4609                 depth = hba->nutrs;
4610         return scsi_change_queue_depth(sdev, depth);
4611 }
4612
4613 /**
4614  * ufshcd_slave_configure - adjust SCSI device configurations
4615  * @sdev: pointer to SCSI device
4616  */
4617 static int ufshcd_slave_configure(struct scsi_device *sdev)
4618 {
4619         struct ufs_hba *hba = shost_priv(sdev->host);
4620         struct request_queue *q = sdev->request_queue;
4621
4622         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4623
4624         if (ufshcd_is_rpm_autosuspend_allowed(hba))
4625                 sdev->rpm_autosuspend = 1;
4626
4627         return 0;
4628 }
4629
4630 /**
4631  * ufshcd_slave_destroy - remove SCSI device configurations
4632  * @sdev: pointer to SCSI device
4633  */
4634 static void ufshcd_slave_destroy(struct scsi_device *sdev)
4635 {
4636         struct ufs_hba *hba;
4637
4638         hba = shost_priv(sdev->host);
4639         /* Drop the reference as it won't be needed anymore */
4640         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4641                 unsigned long flags;
4642
4643                 spin_lock_irqsave(hba->host->host_lock, flags);
4644                 hba->sdev_ufs_device = NULL;
4645                 spin_unlock_irqrestore(hba->host->host_lock, flags);
4646         }
4647 }
4648
4649 /**
4650  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4651  * @lrbp: pointer to local reference block of completed command
4652  * @scsi_status: SCSI command status
4653  *
4654  * Returns value base on SCSI command status
4655  */
4656 static inline int
4657 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4658 {
4659         int result = 0;
4660
4661         switch (scsi_status) {
4662         case SAM_STAT_CHECK_CONDITION:
4663                 ufshcd_copy_sense_data(lrbp);
4664                 /* fallthrough */
4665         case SAM_STAT_GOOD:
4666                 result |= DID_OK << 16 |
4667                           COMMAND_COMPLETE << 8 |
4668                           scsi_status;
4669                 break;
4670         case SAM_STAT_TASK_SET_FULL:
4671         case SAM_STAT_BUSY:
4672         case SAM_STAT_TASK_ABORTED:
4673                 ufshcd_copy_sense_data(lrbp);
4674                 result |= scsi_status;
4675                 break;
4676         default:
4677                 result |= DID_ERROR << 16;
4678                 break;
4679         } /* end of switch */
4680
4681         return result;
4682 }
4683
4684 /**
4685  * ufshcd_transfer_rsp_status - Get overall status of the response
4686  * @hba: per adapter instance
4687  * @lrbp: pointer to local reference block of completed command
4688  *
4689  * Returns result of the command to notify SCSI midlayer
4690  */
4691 static inline int
4692 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4693 {
4694         int result = 0;
4695         int scsi_status;
4696         int ocs;
4697
4698         /* overall command status of utrd */
4699         ocs = ufshcd_get_tr_ocs(lrbp);
4700
4701         switch (ocs) {
4702         case OCS_SUCCESS:
4703                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
4704                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
4705                 switch (result) {
4706                 case UPIU_TRANSACTION_RESPONSE:
4707                         /*
4708                          * get the response UPIU result to extract
4709                          * the SCSI command status
4710                          */
4711                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4712
4713                         /*
4714                          * get the result based on SCSI status response
4715                          * to notify the SCSI midlayer of the command status
4716                          */
4717                         scsi_status = result & MASK_SCSI_STATUS;
4718                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
4719
4720                         /*
4721                          * Currently we are only supporting BKOPs exception
4722                          * events hence we can ignore BKOPs exception event
4723                          * during power management callbacks. BKOPs exception
4724                          * event is not expected to be raised in runtime suspend
4725                          * callback as it allows the urgent bkops.
4726                          * During system suspend, we are anyway forcefully
4727                          * disabling the bkops and if urgent bkops is needed
4728                          * it will be enabled on system resume. Long term
4729                          * solution could be to abort the system suspend if
4730                          * UFS device needs urgent BKOPs.
4731                          */
4732                         if (!hba->pm_op_in_progress &&
4733                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
4734                                 schedule_work(&hba->eeh_work);
4735                         break;
4736                 case UPIU_TRANSACTION_REJECT_UPIU:
4737                         /* TODO: handle Reject UPIU Response */
4738                         result = DID_ERROR << 16;
4739                         dev_err(hba->dev,
4740                                 "Reject UPIU not fully implemented\n");
4741                         break;
4742                 default:
4743                         dev_err(hba->dev,
4744                                 "Unexpected request response code = %x\n",
4745                                 result);
4746                         result = DID_ERROR << 16;
4747                         break;
4748                 }
4749                 break;
4750         case OCS_ABORTED:
4751                 result |= DID_ABORT << 16;
4752                 break;
4753         case OCS_INVALID_COMMAND_STATUS:
4754                 result |= DID_REQUEUE << 16;
4755                 break;
4756         case OCS_INVALID_CMD_TABLE_ATTR:
4757         case OCS_INVALID_PRDT_ATTR:
4758         case OCS_MISMATCH_DATA_BUF_SIZE:
4759         case OCS_MISMATCH_RESP_UPIU_SIZE:
4760         case OCS_PEER_COMM_FAILURE:
4761         case OCS_FATAL_ERROR:
4762         default:
4763                 result |= DID_ERROR << 16;
4764                 dev_err(hba->dev,
4765                                 "OCS error from controller = %x for tag %d\n",
4766                                 ocs, lrbp->task_tag);
4767                 ufshcd_print_host_regs(hba);
4768                 ufshcd_print_host_state(hba);
4769                 break;
4770         } /* end of switch */
4771
4772         if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
4773                 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
4774         return result;
4775 }
4776
4777 /**
4778  * ufshcd_uic_cmd_compl - handle completion of uic command
4779  * @hba: per adapter instance
4780  * @intr_status: interrupt status generated by the controller
4781  *
4782  * Returns
4783  *  IRQ_HANDLED - If interrupt is valid
4784  *  IRQ_NONE    - If invalid interrupt
4785  */
4786 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
4787 {
4788         irqreturn_t retval = IRQ_NONE;
4789
4790         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
4791                 hba->active_uic_cmd->argument2 |=
4792                         ufshcd_get_uic_cmd_result(hba);
4793                 hba->active_uic_cmd->argument3 =
4794                         ufshcd_get_dme_attr_val(hba);
4795                 complete(&hba->active_uic_cmd->done);
4796                 retval = IRQ_HANDLED;
4797         }
4798
4799         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
4800                 complete(hba->uic_async_done);
4801                 retval = IRQ_HANDLED;
4802         }
4803         return retval;
4804 }
4805
4806 /**
4807  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4808  * @hba: per adapter instance
4809  * @completed_reqs: requests to complete
4810  */
4811 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4812                                         unsigned long completed_reqs)
4813 {
4814         struct ufshcd_lrb *lrbp;
4815         struct scsi_cmnd *cmd;
4816         int result;
4817         int index;
4818
4819         for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4820                 lrbp = &hba->lrb[index];
4821                 cmd = lrbp->cmd;
4822                 if (cmd) {
4823                         ufshcd_add_command_trace(hba, index, "complete");
4824                         result = ufshcd_transfer_rsp_status(hba, lrbp);
4825                         scsi_dma_unmap(cmd);
4826                         cmd->result = result;
4827                         /* Mark completed command as NULL in LRB */
4828                         lrbp->cmd = NULL;
4829                         lrbp->compl_time_stamp = ktime_get();
4830                         /* Do not touch lrbp after scsi done */
4831                         cmd->scsi_done(cmd);
4832                         __ufshcd_release(hba);
4833                 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4834                         lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
4835                         lrbp->compl_time_stamp = ktime_get();
4836                         if (hba->dev_cmd.complete) {
4837                                 ufshcd_add_command_trace(hba, index,
4838                                                 "dev_complete");
4839                                 complete(hba->dev_cmd.complete);
4840                         }
4841                 }
4842                 if (ufshcd_is_clkscaling_supported(hba))
4843                         hba->clk_scaling.active_reqs--;
4844         }
4845
4846         /* clear corresponding bits of completed commands */
4847         hba->outstanding_reqs ^= completed_reqs;
4848
4849         ufshcd_clk_scaling_update_busy(hba);
4850 }
4851
4852 /**
4853  * ufshcd_transfer_req_compl - handle SCSI and query command completion
4854  * @hba: per adapter instance
4855  *
4856  * Returns
4857  *  IRQ_HANDLED - If interrupt is valid
4858  *  IRQ_NONE    - If invalid interrupt
4859  */
4860 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
4861 {
4862         unsigned long completed_reqs;
4863         u32 tr_doorbell;
4864
4865         /* Resetting interrupt aggregation counters first and reading the
4866          * DOOR_BELL afterward allows us to handle all the completed requests.
4867          * In order to prevent other interrupts starvation the DB is read once
4868          * after reset. The down side of this solution is the possibility of
4869          * false interrupt if device completes another request after resetting
4870          * aggregation and before reading the DB.
4871          */
4872         if (ufshcd_is_intr_aggr_allowed(hba) &&
4873             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
4874                 ufshcd_reset_intr_aggr(hba);
4875
4876         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4877         completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4878
4879         if (completed_reqs) {
4880                 __ufshcd_transfer_req_compl(hba, completed_reqs);
4881                 return IRQ_HANDLED;
4882         } else {
4883                 return IRQ_NONE;
4884         }
4885 }
4886
4887 /**
4888  * ufshcd_disable_ee - disable exception event
4889  * @hba: per-adapter instance
4890  * @mask: exception event to disable
4891  *
4892  * Disables exception event in the device so that the EVENT_ALERT
4893  * bit is not set.
4894  *
4895  * Returns zero on success, non-zero error value on failure.
4896  */
4897 static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4898 {
4899         int err = 0;
4900         u32 val;
4901
4902         if (!(hba->ee_ctrl_mask & mask))
4903                 goto out;
4904
4905         val = hba->ee_ctrl_mask & ~mask;
4906         val &= MASK_EE_STATUS;
4907         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4908                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4909         if (!err)
4910                 hba->ee_ctrl_mask &= ~mask;
4911 out:
4912         return err;
4913 }
4914
4915 /**
4916  * ufshcd_enable_ee - enable exception event
4917  * @hba: per-adapter instance
4918  * @mask: exception event to enable
4919  *
4920  * Enable corresponding exception event in the device to allow
4921  * device to alert host in critical scenarios.
4922  *
4923  * Returns zero on success, non-zero error value on failure.
4924  */
4925 static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4926 {
4927         int err = 0;
4928         u32 val;
4929
4930         if (hba->ee_ctrl_mask & mask)
4931                 goto out;
4932
4933         val = hba->ee_ctrl_mask | mask;
4934         val &= MASK_EE_STATUS;
4935         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4936                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4937         if (!err)
4938                 hba->ee_ctrl_mask |= mask;
4939 out:
4940         return err;
4941 }
4942
4943 /**
4944  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4945  * @hba: per-adapter instance
4946  *
4947  * Allow device to manage background operations on its own. Enabling
4948  * this might lead to inconsistent latencies during normal data transfers
4949  * as the device is allowed to manage its own way of handling background
4950  * operations.
4951  *
4952  * Returns zero on success, non-zero on failure.
4953  */
4954 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
4955 {
4956         int err = 0;
4957
4958         if (hba->auto_bkops_enabled)
4959                 goto out;
4960
4961         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4962                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
4963         if (err) {
4964                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
4965                                 __func__, err);
4966                 goto out;
4967         }
4968
4969         hba->auto_bkops_enabled = true;
4970         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
4971
4972         /* No need of URGENT_BKOPS exception from the device */
4973         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
4974         if (err)
4975                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
4976                                 __func__, err);
4977 out:
4978         return err;
4979 }
4980
4981 /**
4982  * ufshcd_disable_auto_bkops - block device in doing background operations
4983  * @hba: per-adapter instance
4984  *
4985  * Disabling background operations improves command response latency but
4986  * has drawback of device moving into critical state where the device is
4987  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4988  * host is idle so that BKOPS are managed effectively without any negative
4989  * impacts.
4990  *
4991  * Returns zero on success, non-zero on failure.
4992  */
4993 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
4994 {
4995         int err = 0;
4996
4997         if (!hba->auto_bkops_enabled)
4998                 goto out;
4999
5000         /*
5001          * If host assisted BKOPs is to be enabled, make sure
5002          * urgent bkops exception is allowed.
5003          */
5004         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5005         if (err) {
5006                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5007                                 __func__, err);
5008                 goto out;
5009         }
5010
5011         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5012                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
5013         if (err) {
5014                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5015                                 __func__, err);
5016                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5017                 goto out;
5018         }
5019
5020         hba->auto_bkops_enabled = false;
5021         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5022         hba->is_urgent_bkops_lvl_checked = false;
5023 out:
5024         return err;
5025 }
5026
5027 /**
5028  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5029  * @hba: per adapter instance
5030  *
5031  * After a device reset the device may toggle the BKOPS_EN flag
5032  * to default value. The s/w tracking variables should be updated
5033  * as well. This function would change the auto-bkops state based on
5034  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5035  */
5036 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5037 {
5038         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5039                 hba->auto_bkops_enabled = false;
5040                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5041                 ufshcd_enable_auto_bkops(hba);
5042         } else {
5043                 hba->auto_bkops_enabled = true;
5044                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5045                 ufshcd_disable_auto_bkops(hba);
5046         }
5047         hba->is_urgent_bkops_lvl_checked = false;
5048 }
5049
5050 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5051 {
5052         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5053                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5054 }
5055
5056 /**
5057  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5058  * @hba: per-adapter instance
5059  * @status: bkops_status value
5060  *
5061  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5062  * flag in the device to permit background operations if the device
5063  * bkops_status is greater than or equal to "status" argument passed to
5064  * this function, disable otherwise.
5065  *
5066  * Returns 0 for success, non-zero in case of failure.
5067  *
5068  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5069  * to know whether auto bkops is enabled or disabled after this function
5070  * returns control to it.
5071  */
5072 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5073                              enum bkops_status status)
5074 {
5075         int err;
5076         u32 curr_status = 0;
5077
5078         err = ufshcd_get_bkops_status(hba, &curr_status);
5079         if (err) {
5080                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5081                                 __func__, err);
5082                 goto out;
5083         } else if (curr_status > BKOPS_STATUS_MAX) {
5084                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5085                                 __func__, curr_status);
5086                 err = -EINVAL;
5087                 goto out;
5088         }
5089
5090         if (curr_status >= status)
5091                 err = ufshcd_enable_auto_bkops(hba);
5092         else
5093                 err = ufshcd_disable_auto_bkops(hba);
5094         hba->urgent_bkops_lvl = curr_status;
5095 out:
5096         return err;
5097 }
5098
5099 /**
5100  * ufshcd_urgent_bkops - handle urgent bkops exception event
5101  * @hba: per-adapter instance
5102  *
5103  * Enable fBackgroundOpsEn flag in the device to permit background
5104  * operations.
5105  *
5106  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5107  * and negative error value for any other failure.
5108  */
5109 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5110 {
5111         return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5112 }
5113
5114 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5115 {
5116         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5117                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5118 }
5119
5120 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5121 {
5122         int err;
5123         u32 curr_status = 0;
5124
5125         if (hba->is_urgent_bkops_lvl_checked)
5126                 goto enable_auto_bkops;
5127
5128         err = ufshcd_get_bkops_status(hba, &curr_status);
5129         if (err) {
5130                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5131                                 __func__, err);
5132                 goto out;
5133         }
5134
5135         /*
5136          * We are seeing that some devices are raising the urgent bkops
5137          * exception events even when BKOPS status doesn't indicate performace
5138          * impacted or critical. Handle these device by determining their urgent
5139          * bkops status at runtime.
5140          */
5141         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5142                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5143                                 __func__, curr_status);
5144                 /* update the current status as the urgent bkops level */
5145                 hba->urgent_bkops_lvl = curr_status;
5146                 hba->is_urgent_bkops_lvl_checked = true;
5147         }
5148
5149 enable_auto_bkops:
5150         err = ufshcd_enable_auto_bkops(hba);
5151 out:
5152         if (err < 0)
5153                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5154                                 __func__, err);
5155 }
5156
5157 /**
5158  * ufshcd_exception_event_handler - handle exceptions raised by device
5159  * @work: pointer to work data
5160  *
5161  * Read bExceptionEventStatus attribute from the device and handle the
5162  * exception event accordingly.
5163  */
5164 static void ufshcd_exception_event_handler(struct work_struct *work)
5165 {
5166         struct ufs_hba *hba;
5167         int err;
5168         u32 status = 0;
5169         hba = container_of(work, struct ufs_hba, eeh_work);
5170
5171         pm_runtime_get_sync(hba->dev);
5172         ufshcd_scsi_block_requests(hba);
5173         err = ufshcd_get_ee_status(hba, &status);
5174         if (err) {
5175                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5176                                 __func__, err);
5177                 goto out;
5178         }
5179
5180         status &= hba->ee_ctrl_mask;
5181
5182         if (status & MASK_EE_URGENT_BKOPS)
5183                 ufshcd_bkops_exception_event_handler(hba);
5184
5185 out:
5186         ufshcd_scsi_unblock_requests(hba);
5187         pm_runtime_put_sync(hba->dev);
5188         return;
5189 }
5190
5191 /* Complete requests that have door-bell cleared */
5192 static void ufshcd_complete_requests(struct ufs_hba *hba)
5193 {
5194         ufshcd_transfer_req_compl(hba);
5195         ufshcd_tmc_handler(hba);
5196 }
5197
5198 /**
5199  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5200  *                              to recover from the DL NAC errors or not.
5201  * @hba: per-adapter instance
5202  *
5203  * Returns true if error handling is required, false otherwise
5204  */
5205 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5206 {
5207         unsigned long flags;
5208         bool err_handling = true;
5209
5210         spin_lock_irqsave(hba->host->host_lock, flags);
5211         /*
5212          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5213          * device fatal error and/or DL NAC & REPLAY timeout errors.
5214          */
5215         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5216                 goto out;
5217
5218         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5219             ((hba->saved_err & UIC_ERROR) &&
5220              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5221                 goto out;
5222
5223         if ((hba->saved_err & UIC_ERROR) &&
5224             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5225                 int err;
5226                 /*
5227                  * wait for 50ms to see if we can get any other errors or not.
5228                  */
5229                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5230                 msleep(50);
5231                 spin_lock_irqsave(hba->host->host_lock, flags);
5232
5233                 /*
5234                  * now check if we have got any other severe errors other than
5235                  * DL NAC error?
5236                  */
5237                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5238                     ((hba->saved_err & UIC_ERROR) &&
5239                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5240                         goto out;
5241
5242                 /*
5243                  * As DL NAC is the only error received so far, send out NOP
5244                  * command to confirm if link is still active or not.
5245                  *   - If we don't get any response then do error recovery.
5246                  *   - If we get response then clear the DL NAC error bit.
5247                  */
5248
5249                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5250                 err = ufshcd_verify_dev_init(hba);
5251                 spin_lock_irqsave(hba->host->host_lock, flags);
5252
5253                 if (err)
5254                         goto out;
5255
5256                 /* Link seems to be alive hence ignore the DL NAC errors */
5257                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5258                         hba->saved_err &= ~UIC_ERROR;
5259                 /* clear NAC error */
5260                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5261                 if (!hba->saved_uic_err) {
5262                         err_handling = false;
5263                         goto out;
5264                 }
5265         }
5266 out:
5267         spin_unlock_irqrestore(hba->host->host_lock, flags);
5268         return err_handling;
5269 }
5270
5271 /**
5272  * ufshcd_err_handler - handle UFS errors that require s/w attention
5273  * @work: pointer to work structure
5274  */
5275 static void ufshcd_err_handler(struct work_struct *work)
5276 {
5277         struct ufs_hba *hba;
5278         unsigned long flags;
5279         u32 err_xfer = 0;
5280         u32 err_tm = 0;
5281         int err = 0;
5282         int tag;
5283         bool needs_reset = false;
5284
5285         hba = container_of(work, struct ufs_hba, eh_work);
5286
5287         pm_runtime_get_sync(hba->dev);
5288         ufshcd_hold(hba, false);
5289
5290         spin_lock_irqsave(hba->host->host_lock, flags);
5291         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
5292                 goto out;
5293
5294         hba->ufshcd_state = UFSHCD_STATE_RESET;
5295         ufshcd_set_eh_in_progress(hba);
5296
5297         /* Complete requests that have door-bell cleared by h/w */
5298         ufshcd_complete_requests(hba);
5299
5300         if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5301                 bool ret;
5302
5303                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5304                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5305                 ret = ufshcd_quirk_dl_nac_errors(hba);
5306                 spin_lock_irqsave(hba->host->host_lock, flags);
5307                 if (!ret)
5308                         goto skip_err_handling;
5309         }
5310         if ((hba->saved_err & INT_FATAL_ERRORS) ||
5311             (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
5312             ((hba->saved_err & UIC_ERROR) &&
5313             (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5314                                    UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5315                                    UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5316                 needs_reset = true;
5317
5318         /*
5319          * if host reset is required then skip clearing the pending
5320          * transfers forcefully because they will get cleared during
5321          * host reset and restore
5322          */
5323         if (needs_reset)
5324                 goto skip_pending_xfer_clear;
5325
5326         /* release lock as clear command might sleep */
5327         spin_unlock_irqrestore(hba->host->host_lock, flags);
5328         /* Clear pending transfer requests */
5329         for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5330                 if (ufshcd_clear_cmd(hba, tag)) {
5331                         err_xfer = true;
5332                         goto lock_skip_pending_xfer_clear;
5333                 }
5334         }
5335
5336         /* Clear pending task management requests */
5337         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5338                 if (ufshcd_clear_tm_cmd(hba, tag)) {
5339                         err_tm = true;
5340                         goto lock_skip_pending_xfer_clear;
5341                 }
5342         }
5343
5344 lock_skip_pending_xfer_clear:
5345         spin_lock_irqsave(hba->host->host_lock, flags);
5346
5347         /* Complete the requests that are cleared by s/w */
5348         ufshcd_complete_requests(hba);
5349
5350         if (err_xfer || err_tm)
5351                 needs_reset = true;
5352
5353 skip_pending_xfer_clear:
5354         /* Fatal errors need reset */
5355         if (needs_reset) {
5356                 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5357
5358                 /*
5359                  * ufshcd_reset_and_restore() does the link reinitialization
5360                  * which will need atleast one empty doorbell slot to send the
5361                  * device management commands (NOP and query commands).
5362                  * If there is no slot empty at this moment then free up last
5363                  * slot forcefully.
5364                  */
5365                 if (hba->outstanding_reqs == max_doorbells)
5366                         __ufshcd_transfer_req_compl(hba,
5367                                                     (1UL << (hba->nutrs - 1)));
5368
5369                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5370                 err = ufshcd_reset_and_restore(hba);
5371                 spin_lock_irqsave(hba->host->host_lock, flags);
5372                 if (err) {
5373                         dev_err(hba->dev, "%s: reset and restore failed\n",
5374                                         __func__);
5375                         hba->ufshcd_state = UFSHCD_STATE_ERROR;
5376                 }
5377                 /*
5378                  * Inform scsi mid-layer that we did reset and allow to handle
5379                  * Unit Attention properly.
5380                  */
5381                 scsi_report_bus_reset(hba->host, 0);
5382                 hba->saved_err = 0;
5383                 hba->saved_uic_err = 0;
5384         }
5385
5386 skip_err_handling:
5387         if (!needs_reset) {
5388                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5389                 if (hba->saved_err || hba->saved_uic_err)
5390                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5391                             __func__, hba->saved_err, hba->saved_uic_err);
5392         }
5393
5394         ufshcd_clear_eh_in_progress(hba);
5395
5396 out:
5397         spin_unlock_irqrestore(hba->host->host_lock, flags);
5398         ufshcd_scsi_unblock_requests(hba);
5399         ufshcd_release(hba);
5400         pm_runtime_put_sync(hba->dev);
5401 }
5402
5403 /**
5404  * ufshcd_update_uic_error - check and set fatal UIC error flags.
5405  * @hba: per-adapter instance
5406  *
5407  * Returns
5408  *  IRQ_HANDLED - If interrupt is valid
5409  *  IRQ_NONE    - If invalid interrupt
5410  */
5411 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
5412 {
5413         u32 reg;
5414         irqreturn_t retval = IRQ_NONE;
5415
5416         /* PHY layer lane error */
5417         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5418         /* Ignore LINERESET indication, as this is not an error */
5419         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
5420             (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
5421                 /*
5422                  * To know whether this error is fatal or not, DB timeout
5423                  * must be checked but this error is handled separately.
5424                  */
5425                 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
5426                 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
5427                 retval |= IRQ_HANDLED;
5428         }
5429
5430         /* PA_INIT_ERROR is fatal and needs UIC reset */
5431         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
5432         if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5433             (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
5434                 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
5435
5436                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5437                         hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5438                 else if (hba->dev_quirks &
5439                                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5440                         if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5441                                 hba->uic_error |=
5442                                         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5443                         else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5444                                 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5445                 }
5446                 retval |= IRQ_HANDLED;
5447         }
5448
5449         /* UIC NL/TL/DME errors needs software retry */
5450         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
5451         if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5452             (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
5453                 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
5454                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
5455                 retval |= IRQ_HANDLED;
5456         }
5457
5458         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
5459         if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5460             (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
5461                 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
5462                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
5463                 retval |= IRQ_HANDLED;
5464         }
5465
5466         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
5467         if ((reg & UIC_DME_ERROR) &&
5468             (reg & UIC_DME_ERROR_CODE_MASK)) {
5469                 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
5470                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
5471                 retval |= IRQ_HANDLED;
5472         }
5473
5474         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5475                         __func__, hba->uic_error);
5476         return retval;
5477 }
5478
5479 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5480                                          u32 intr_mask)
5481 {
5482         if (!ufshcd_is_auto_hibern8_supported(hba))
5483                 return false;
5484
5485         if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5486                 return false;
5487
5488         if (hba->active_uic_cmd &&
5489             (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5490             hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5491                 return false;
5492
5493         return true;
5494 }
5495
5496 /**
5497  * ufshcd_check_errors - Check for errors that need s/w attention
5498  * @hba: per-adapter instance
5499  *
5500  * Returns
5501  *  IRQ_HANDLED - If interrupt is valid
5502  *  IRQ_NONE    - If invalid interrupt
5503  */
5504 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
5505 {
5506         bool queue_eh_work = false;
5507         irqreturn_t retval = IRQ_NONE;
5508
5509         if (hba->errors & INT_FATAL_ERRORS) {
5510                 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
5511                 queue_eh_work = true;
5512         }
5513
5514         if (hba->errors & UIC_ERROR) {
5515                 hba->uic_error = 0;
5516                 retval = ufshcd_update_uic_error(hba);
5517                 if (hba->uic_error)
5518                         queue_eh_work = true;
5519         }
5520
5521         if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
5522                 dev_err(hba->dev,
5523                         "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
5524                         __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
5525                         "Enter" : "Exit",
5526                         hba->errors, ufshcd_get_upmcrs(hba));
5527                 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
5528                                        hba->errors);
5529                 queue_eh_work = true;
5530         }
5531
5532         if (queue_eh_work) {
5533                 /*
5534                  * update the transfer error masks to sticky bits, let's do this
5535                  * irrespective of current ufshcd_state.
5536                  */
5537                 hba->saved_err |= hba->errors;
5538                 hba->saved_uic_err |= hba->uic_error;
5539
5540                 /* handle fatal errors only when link is functional */
5541                 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5542                         /* block commands from scsi mid-layer */
5543                         ufshcd_scsi_block_requests(hba);
5544
5545                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
5546
5547                         /* dump controller state before resetting */
5548                         if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5549                                 bool pr_prdt = !!(hba->saved_err &
5550                                                 SYSTEM_BUS_FATAL_ERROR);
5551
5552                                 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5553                                         __func__, hba->saved_err,
5554                                         hba->saved_uic_err);
5555
5556                                 ufshcd_print_host_regs(hba);
5557                                 ufshcd_print_pwr_info(hba);
5558                                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5559                                 ufshcd_print_trs(hba, hba->outstanding_reqs,
5560                                                         pr_prdt);
5561                         }
5562                         schedule_work(&hba->eh_work);
5563                 }
5564                 retval |= IRQ_HANDLED;
5565         }
5566         /*
5567          * if (!queue_eh_work) -
5568          * Other errors are either non-fatal where host recovers
5569          * itself without s/w intervention or errors that will be
5570          * handled by the SCSI core layer.
5571          */
5572         return retval;
5573 }
5574
5575 struct ctm_info {
5576         struct ufs_hba  *hba;
5577         unsigned long   pending;
5578         unsigned int    ncpl;
5579 };
5580
5581 static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
5582 {
5583         struct ctm_info *const ci = priv;
5584         struct completion *c;
5585
5586         WARN_ON_ONCE(reserved);
5587         if (test_bit(req->tag, &ci->pending))
5588                 return true;
5589         ci->ncpl++;
5590         c = req->end_io_data;
5591         if (c)
5592                 complete(c);
5593         return true;
5594 }
5595
5596 /**
5597  * ufshcd_tmc_handler - handle task management function completion
5598  * @hba: per adapter instance
5599  *
5600  * Returns
5601  *  IRQ_HANDLED - If interrupt is valid
5602  *  IRQ_NONE    - If invalid interrupt
5603  */
5604 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
5605 {
5606         struct request_queue *q = hba->tmf_queue;
5607         struct ctm_info ci = {
5608                 .hba     = hba,
5609                 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
5610         };
5611
5612         blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
5613         return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
5614 }
5615
5616 /**
5617  * ufshcd_sl_intr - Interrupt service routine
5618  * @hba: per adapter instance
5619  * @intr_status: contains interrupts generated by the controller
5620  *
5621  * Returns
5622  *  IRQ_HANDLED - If interrupt is valid
5623  *  IRQ_NONE    - If invalid interrupt
5624  */
5625 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
5626 {
5627         irqreturn_t retval = IRQ_NONE;
5628
5629         hba->errors = UFSHCD_ERROR_MASK & intr_status;
5630
5631         if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5632                 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5633
5634         if (hba->errors)
5635                 retval |= ufshcd_check_errors(hba);
5636
5637         if (intr_status & UFSHCD_UIC_MASK)
5638                 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
5639
5640         if (intr_status & UTP_TASK_REQ_COMPL)
5641                 retval |= ufshcd_tmc_handler(hba);
5642
5643         if (intr_status & UTP_TRANSFER_REQ_COMPL)
5644                 retval |= ufshcd_transfer_req_compl(hba);
5645
5646         return retval;
5647 }
5648
5649 /**
5650  * ufshcd_intr - Main interrupt service routine
5651  * @irq: irq number
5652  * @__hba: pointer to adapter instance
5653  *
5654  * Returns
5655  *  IRQ_HANDLED - If interrupt is valid
5656  *  IRQ_NONE    - If invalid interrupt
5657  */
5658 static irqreturn_t ufshcd_intr(int irq, void *__hba)
5659 {
5660         u32 intr_status, enabled_intr_status;
5661         irqreturn_t retval = IRQ_NONE;
5662         struct ufs_hba *hba = __hba;
5663         int retries = hba->nutrs;
5664
5665         spin_lock(hba->host->host_lock);
5666         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5667
5668         /*
5669          * There could be max of hba->nutrs reqs in flight and in worst case
5670          * if the reqs get finished 1 by 1 after the interrupt status is
5671          * read, make sure we handle them by checking the interrupt status
5672          * again in a loop until we process all of the reqs before returning.
5673          */
5674         do {
5675                 enabled_intr_status =
5676                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5677                 if (intr_status)
5678                         ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
5679                 if (enabled_intr_status)
5680                         retval |= ufshcd_sl_intr(hba, enabled_intr_status);
5681
5682                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5683         } while (intr_status && --retries);
5684
5685         if (retval == IRQ_NONE) {
5686                 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
5687                                         __func__, intr_status);
5688                 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
5689         }
5690
5691         spin_unlock(hba->host->host_lock);
5692         return retval;
5693 }
5694
5695 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5696 {
5697         int err = 0;
5698         u32 mask = 1 << tag;
5699         unsigned long flags;
5700
5701         if (!test_bit(tag, &hba->outstanding_tasks))
5702                 goto out;
5703
5704         spin_lock_irqsave(hba->host->host_lock, flags);
5705         ufshcd_utmrl_clear(hba, tag);
5706         spin_unlock_irqrestore(hba->host->host_lock, flags);
5707
5708         /* poll for max. 1 sec to clear door bell register by h/w */
5709         err = ufshcd_wait_for_register(hba,
5710                         REG_UTP_TASK_REQ_DOOR_BELL,
5711                         mask, 0, 1000, 1000, true);
5712 out:
5713         return err;
5714 }
5715
5716 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
5717                 struct utp_task_req_desc *treq, u8 tm_function)
5718 {
5719         struct request_queue *q = hba->tmf_queue;
5720         struct Scsi_Host *host = hba->host;
5721         DECLARE_COMPLETION_ONSTACK(wait);
5722         struct request *req;
5723         unsigned long flags;
5724         int free_slot, task_tag, err;
5725
5726         /*
5727          * Get free slot, sleep if slots are unavailable.
5728          * Even though we use wait_event() which sleeps indefinitely,
5729          * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5730          */
5731         req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
5732         req->end_io_data = &wait;
5733         free_slot = req->tag;
5734         WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
5735         ufshcd_hold(hba, false);
5736
5737         spin_lock_irqsave(host->host_lock, flags);
5738         task_tag = hba->nutrs + free_slot;
5739
5740         treq->req_header.dword_0 |= cpu_to_be32(task_tag);
5741
5742         memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
5743         ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5744
5745         /* send command to the controller */
5746         __set_bit(free_slot, &hba->outstanding_tasks);
5747
5748         /* Make sure descriptors are ready before ringing the task doorbell */
5749         wmb();
5750
5751         ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
5752         /* Make sure that doorbell is committed immediately */
5753         wmb();
5754
5755         spin_unlock_irqrestore(host->host_lock, flags);
5756
5757         ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5758
5759         /* wait until the task management command is completed */
5760         err = wait_for_completion_io_timeout(&wait,
5761                         msecs_to_jiffies(TM_CMD_TIMEOUT));
5762         if (!err) {
5763                 /*
5764                  * Make sure that ufshcd_compl_tm() does not trigger a
5765                  * use-after-free.
5766                  */
5767                 req->end_io_data = NULL;
5768                 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
5769                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5770                                 __func__, tm_function);
5771                 if (ufshcd_clear_tm_cmd(hba, free_slot))
5772                         dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5773                                         __func__, free_slot);
5774                 err = -ETIMEDOUT;
5775         } else {
5776                 err = 0;
5777                 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
5778
5779                 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
5780         }
5781
5782         spin_lock_irqsave(hba->host->host_lock, flags);
5783         __clear_bit(free_slot, &hba->outstanding_tasks);
5784         spin_unlock_irqrestore(hba->host->host_lock, flags);
5785
5786         blk_put_request(req);
5787
5788         ufshcd_release(hba);
5789         return err;
5790 }
5791
5792 /**
5793  * ufshcd_issue_tm_cmd - issues task management commands to controller
5794  * @hba: per adapter instance
5795  * @lun_id: LUN ID to which TM command is sent
5796  * @task_id: task ID to which the TM command is applicable
5797  * @tm_function: task management function opcode
5798  * @tm_response: task management service response return value
5799  *
5800  * Returns non-zero value on error, zero on success.
5801  */
5802 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5803                 u8 tm_function, u8 *tm_response)
5804 {
5805         struct utp_task_req_desc treq = { { 0 }, };
5806         int ocs_value, err;
5807
5808         /* Configure task request descriptor */
5809         treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5810         treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5811
5812         /* Configure task request UPIU */
5813         treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
5814                                   cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
5815         treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
5816
5817         /*
5818          * The host shall provide the same value for LUN field in the basic
5819          * header and for Input Parameter.
5820          */
5821         treq.input_param1 = cpu_to_be32(lun_id);
5822         treq.input_param2 = cpu_to_be32(task_id);
5823
5824         err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
5825         if (err == -ETIMEDOUT)
5826                 return err;
5827
5828         ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
5829         if (ocs_value != OCS_SUCCESS)
5830                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
5831                                 __func__, ocs_value);
5832         else if (tm_response)
5833                 *tm_response = be32_to_cpu(treq.output_param1) &
5834                                 MASK_TM_SERVICE_RESP;
5835         return err;
5836 }
5837
5838 /**
5839  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5840  * @hba:        per-adapter instance
5841  * @req_upiu:   upiu request
5842  * @rsp_upiu:   upiu reply
5843  * @desc_buff:  pointer to descriptor buffer, NULL if NA
5844  * @buff_len:   descriptor size, 0 if NA
5845  * @cmd_type:   specifies the type (NOP, Query...)
5846  * @desc_op:    descriptor operation
5847  *
5848  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5849  * Therefore, it "rides" the device management infrastructure: uses its tag and
5850  * tasks work queues.
5851  *
5852  * Since there is only one available tag for device management commands,
5853  * the caller is expected to hold the hba->dev_cmd.lock mutex.
5854  */
5855 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
5856                                         struct utp_upiu_req *req_upiu,
5857                                         struct utp_upiu_req *rsp_upiu,
5858                                         u8 *desc_buff, int *buff_len,
5859                                         enum dev_cmd_type cmd_type,
5860                                         enum query_opcode desc_op)
5861 {
5862         struct request_queue *q = hba->cmd_queue;
5863         struct request *req;
5864         struct ufshcd_lrb *lrbp;
5865         int err = 0;
5866         int tag;
5867         struct completion wait;
5868         unsigned long flags;
5869         u32 upiu_flags;
5870
5871         down_read(&hba->clk_scaling_lock);
5872
5873         req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
5874         if (IS_ERR(req)) {
5875                 err = PTR_ERR(req);
5876                 goto out_unlock;
5877         }
5878         tag = req->tag;
5879         WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
5880
5881         init_completion(&wait);
5882         lrbp = &hba->lrb[tag];
5883         WARN_ON(lrbp->cmd);
5884
5885         lrbp->cmd = NULL;
5886         lrbp->sense_bufflen = 0;
5887         lrbp->sense_buffer = NULL;
5888         lrbp->task_tag = tag;
5889         lrbp->lun = 0;
5890         lrbp->intr_cmd = true;
5891         hba->dev_cmd.type = cmd_type;
5892
5893         switch (hba->ufs_version) {
5894         case UFSHCI_VERSION_10:
5895         case UFSHCI_VERSION_11:
5896                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
5897                 break;
5898         default:
5899                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
5900                 break;
5901         }
5902
5903         /* update the task tag in the request upiu */
5904         req_upiu->header.dword_0 |= cpu_to_be32(tag);
5905
5906         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
5907
5908         /* just copy the upiu request as it is */
5909         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
5910         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
5911                 /* The Data Segment Area is optional depending upon the query
5912                  * function value. for WRITE DESCRIPTOR, the data segment
5913                  * follows right after the tsf.
5914                  */
5915                 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
5916                 *buff_len = 0;
5917         }
5918
5919         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
5920
5921         hba->dev_cmd.complete = &wait;
5922
5923         /* Make sure descriptors are ready before ringing the doorbell */
5924         wmb();
5925         spin_lock_irqsave(hba->host->host_lock, flags);
5926         ufshcd_send_command(hba, tag);
5927         spin_unlock_irqrestore(hba->host->host_lock, flags);
5928
5929         /*
5930          * ignore the returning value here - ufshcd_check_query_response is
5931          * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5932          * read the response directly ignoring all errors.
5933          */
5934         ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
5935
5936         /* just copy the upiu response as it is */
5937         memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
5938         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
5939                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
5940                 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
5941                                MASK_QUERY_DATA_SEG_LEN;
5942
5943                 if (*buff_len >= resp_len) {
5944                         memcpy(desc_buff, descp, resp_len);
5945                         *buff_len = resp_len;
5946                 } else {
5947                         dev_warn(hba->dev,
5948                                  "%s: rsp size %d is bigger than buffer size %d",
5949                                  __func__, resp_len, *buff_len);
5950                         *buff_len = 0;
5951                         err = -EINVAL;
5952                 }
5953         }
5954
5955         blk_put_request(req);
5956 out_unlock:
5957         up_read(&hba->clk_scaling_lock);
5958         return err;
5959 }
5960
5961 /**
5962  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5963  * @hba:        per-adapter instance
5964  * @req_upiu:   upiu request
5965  * @rsp_upiu:   upiu reply - only 8 DW as we do not support scsi commands
5966  * @msgcode:    message code, one of UPIU Transaction Codes Initiator to Target
5967  * @desc_buff:  pointer to descriptor buffer, NULL if NA
5968  * @buff_len:   descriptor size, 0 if NA
5969  * @desc_op:    descriptor operation
5970  *
5971  * Supports UTP Transfer requests (nop and query), and UTP Task
5972  * Management requests.
5973  * It is up to the caller to fill the upiu conent properly, as it will
5974  * be copied without any further input validations.
5975  */
5976 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
5977                              struct utp_upiu_req *req_upiu,
5978                              struct utp_upiu_req *rsp_upiu,
5979                              int msgcode,
5980                              u8 *desc_buff, int *buff_len,
5981                              enum query_opcode desc_op)
5982 {
5983         int err;
5984         enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
5985         struct utp_task_req_desc treq = { { 0 }, };
5986         int ocs_value;
5987         u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
5988
5989         switch (msgcode) {
5990         case UPIU_TRANSACTION_NOP_OUT:
5991                 cmd_type = DEV_CMD_TYPE_NOP;
5992                 /* fall through */
5993         case UPIU_TRANSACTION_QUERY_REQ:
5994                 ufshcd_hold(hba, false);
5995                 mutex_lock(&hba->dev_cmd.lock);
5996                 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
5997                                                    desc_buff, buff_len,
5998                                                    cmd_type, desc_op);
5999                 mutex_unlock(&hba->dev_cmd.lock);
6000                 ufshcd_release(hba);
6001
6002                 break;
6003         case UPIU_TRANSACTION_TASK_REQ:
6004                 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6005                 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6006
6007                 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6008
6009                 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6010                 if (err == -ETIMEDOUT)
6011                         break;
6012
6013                 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6014                 if (ocs_value != OCS_SUCCESS) {
6015                         dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6016                                 ocs_value);
6017                         break;
6018                 }
6019
6020                 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6021
6022                 break;
6023         default:
6024                 err = -EINVAL;
6025
6026                 break;
6027         }
6028
6029         return err;
6030 }
6031
6032 /**
6033  * ufshcd_eh_device_reset_handler - device reset handler registered to
6034  *                                    scsi layer.
6035  * @cmd: SCSI command pointer
6036  *
6037  * Returns SUCCESS/FAILED
6038  */
6039 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
6040 {
6041         struct Scsi_Host *host;
6042         struct ufs_hba *hba;
6043         unsigned int tag;
6044         u32 pos;
6045         int err;
6046         u8 resp = 0xF;
6047         struct ufshcd_lrb *lrbp;
6048         unsigned long flags;
6049
6050         host = cmd->device->host;
6051         hba = shost_priv(host);
6052         tag = cmd->request->tag;
6053
6054         lrbp = &hba->lrb[tag];
6055         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6056         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6057                 if (!err)
6058                         err = resp;
6059                 goto out;
6060         }
6061
6062         /* clear the commands that were pending for corresponding LUN */
6063         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6064                 if (hba->lrb[pos].lun == lrbp->lun) {
6065                         err = ufshcd_clear_cmd(hba, pos);
6066                         if (err)
6067                                 break;
6068                 }
6069         }
6070         spin_lock_irqsave(host->host_lock, flags);
6071         ufshcd_transfer_req_compl(hba);
6072         spin_unlock_irqrestore(host->host_lock, flags);
6073
6074 out:
6075         hba->req_abort_count = 0;
6076         ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
6077         if (!err) {
6078                 err = SUCCESS;
6079         } else {
6080                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6081                 err = FAILED;
6082         }
6083         return err;
6084 }
6085
6086 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6087 {
6088         struct ufshcd_lrb *lrbp;
6089         int tag;
6090
6091         for_each_set_bit(tag, &bitmap, hba->nutrs) {
6092                 lrbp = &hba->lrb[tag];
6093                 lrbp->req_abort_skip = true;
6094         }
6095 }
6096
6097 /**
6098  * ufshcd_abort - abort a specific command
6099  * @cmd: SCSI command pointer
6100  *
6101  * Abort the pending command in device by sending UFS_ABORT_TASK task management
6102  * command, and in host controller by clearing the door-bell register. There can
6103  * be race between controller sending the command to the device while abort is
6104  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6105  * really issued and then try to abort it.
6106  *
6107  * Returns SUCCESS/FAILED
6108  */
6109 static int ufshcd_abort(struct scsi_cmnd *cmd)
6110 {
6111         struct Scsi_Host *host;
6112         struct ufs_hba *hba;
6113         unsigned long flags;
6114         unsigned int tag;
6115         int err = 0;
6116         int poll_cnt;
6117         u8 resp = 0xF;
6118         struct ufshcd_lrb *lrbp;
6119         u32 reg;
6120
6121         host = cmd->device->host;
6122         hba = shost_priv(host);
6123         tag = cmd->request->tag;
6124         lrbp = &hba->lrb[tag];
6125         if (!ufshcd_valid_tag(hba, tag)) {
6126                 dev_err(hba->dev,
6127                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6128                         __func__, tag, cmd, cmd->request);
6129                 BUG();
6130         }
6131
6132         /*
6133          * Task abort to the device W-LUN is illegal. When this command
6134          * will fail, due to spec violation, scsi err handling next step
6135          * will be to send LU reset which, again, is a spec violation.
6136          * To avoid these unnecessary/illegal step we skip to the last error
6137          * handling stage: reset and restore.
6138          */
6139         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6140                 return ufshcd_eh_host_reset_handler(cmd);
6141
6142         ufshcd_hold(hba, false);
6143         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6144         /* If command is already aborted/completed, return SUCCESS */
6145         if (!(test_bit(tag, &hba->outstanding_reqs))) {
6146                 dev_err(hba->dev,
6147                         "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6148                         __func__, tag, hba->outstanding_reqs, reg);
6149                 goto out;
6150         }
6151
6152         if (!(reg & (1 << tag))) {
6153                 dev_err(hba->dev,
6154                 "%s: cmd was completed, but without a notifying intr, tag = %d",
6155                 __func__, tag);
6156         }
6157
6158         /* Print Transfer Request of aborted task */
6159         dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
6160
6161         /*
6162          * Print detailed info about aborted request.
6163          * As more than one request might get aborted at the same time,
6164          * print full information only for the first aborted request in order
6165          * to reduce repeated printouts. For other aborted requests only print
6166          * basic details.
6167          */
6168         scsi_print_command(hba->lrb[tag].cmd);
6169         if (!hba->req_abort_count) {
6170                 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
6171                 ufshcd_print_host_regs(hba);
6172                 ufshcd_print_host_state(hba);
6173                 ufshcd_print_pwr_info(hba);
6174                 ufshcd_print_trs(hba, 1 << tag, true);
6175         } else {
6176                 ufshcd_print_trs(hba, 1 << tag, false);
6177         }
6178         hba->req_abort_count++;
6179
6180         /* Skip task abort in case previous aborts failed and report failure */
6181         if (lrbp->req_abort_skip) {
6182                 err = -EIO;
6183                 goto out;
6184         }
6185
6186         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6187                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6188                                 UFS_QUERY_TASK, &resp);
6189                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6190                         /* cmd pending in the device */
6191                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6192                                 __func__, tag);
6193                         break;
6194                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6195                         /*
6196                          * cmd not pending in the device, check if it is
6197                          * in transition.
6198                          */
6199                         dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6200                                 __func__, tag);
6201                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6202                         if (reg & (1 << tag)) {
6203                                 /* sleep for max. 200us to stabilize */
6204                                 usleep_range(100, 200);
6205                                 continue;
6206                         }
6207                         /* command completed already */
6208                         dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6209                                 __func__, tag);
6210                         goto out;
6211                 } else {
6212                         dev_err(hba->dev,
6213                                 "%s: no response from device. tag = %d, err %d\n",
6214                                 __func__, tag, err);
6215                         if (!err)
6216                                 err = resp; /* service response error */
6217                         goto out;
6218                 }
6219         }
6220
6221         if (!poll_cnt) {
6222                 err = -EBUSY;
6223                 goto out;
6224         }
6225
6226         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6227                         UFS_ABORT_TASK, &resp);
6228         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6229                 if (!err) {
6230                         err = resp; /* service response error */
6231                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6232                                 __func__, tag, err);
6233                 }
6234                 goto out;
6235         }
6236
6237         err = ufshcd_clear_cmd(hba, tag);
6238         if (err) {
6239                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6240                         __func__, tag, err);
6241                 goto out;
6242         }
6243
6244         scsi_dma_unmap(cmd);
6245
6246         spin_lock_irqsave(host->host_lock, flags);
6247         ufshcd_outstanding_req_clear(hba, tag);
6248         hba->lrb[tag].cmd = NULL;
6249         spin_unlock_irqrestore(host->host_lock, flags);
6250
6251 out:
6252         if (!err) {
6253                 err = SUCCESS;
6254         } else {
6255                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6256                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
6257                 err = FAILED;
6258         }
6259
6260         /*
6261          * This ufshcd_release() corresponds to the original scsi cmd that got
6262          * aborted here (as we won't get any IRQ for it).
6263          */
6264         ufshcd_release(hba);
6265         return err;
6266 }
6267
6268 /**
6269  * ufshcd_host_reset_and_restore - reset and restore host controller
6270  * @hba: per-adapter instance
6271  *
6272  * Note that host controller reset may issue DME_RESET to
6273  * local and remote (device) Uni-Pro stack and the attributes
6274  * are reset to default state.
6275  *
6276  * Returns zero on success, non-zero on failure
6277  */
6278 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6279 {
6280         int err;
6281         unsigned long flags;
6282
6283         /*
6284          * Stop the host controller and complete the requests
6285          * cleared by h/w
6286          */
6287         spin_lock_irqsave(hba->host->host_lock, flags);
6288         ufshcd_hba_stop(hba, false);
6289         hba->silence_err_logs = true;
6290         ufshcd_complete_requests(hba);
6291         hba->silence_err_logs = false;
6292         spin_unlock_irqrestore(hba->host->host_lock, flags);
6293
6294         /* scale up clocks to max frequency before full reinitialization */
6295         ufshcd_scale_clks(hba, true);
6296
6297         err = ufshcd_hba_enable(hba);
6298         if (err)
6299                 goto out;
6300
6301         /* Establish the link again and restore the device */
6302         err = ufshcd_probe_hba(hba, false);
6303
6304         if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
6305                 err = -EIO;
6306 out:
6307         if (err)
6308                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
6309         ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
6310         return err;
6311 }
6312
6313 /**
6314  * ufshcd_reset_and_restore - reset and re-initialize host/device
6315  * @hba: per-adapter instance
6316  *
6317  * Reset and recover device, host and re-establish link. This
6318  * is helpful to recover the communication in fatal error conditions.
6319  *
6320  * Returns zero on success, non-zero on failure
6321  */
6322 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6323 {
6324         int err = 0;
6325         int retries = MAX_HOST_RESET_RETRIES;
6326
6327         do {
6328                 /* Reset the attached device */
6329                 ufshcd_vops_device_reset(hba);
6330
6331                 err = ufshcd_host_reset_and_restore(hba);
6332         } while (err && --retries);
6333
6334         return err;
6335 }
6336
6337 /**
6338  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6339  * @cmd: SCSI command pointer
6340  *
6341  * Returns SUCCESS/FAILED
6342  */
6343 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6344 {
6345         int err;
6346         unsigned long flags;
6347         struct ufs_hba *hba;
6348
6349         hba = shost_priv(cmd->device->host);
6350
6351         ufshcd_hold(hba, false);
6352         /*
6353          * Check if there is any race with fatal error handling.
6354          * If so, wait for it to complete. Even though fatal error
6355          * handling does reset and restore in some cases, don't assume
6356          * anything out of it. We are just avoiding race here.
6357          */
6358         do {
6359                 spin_lock_irqsave(hba->host->host_lock, flags);
6360                 if (!(work_pending(&hba->eh_work) ||
6361                             hba->ufshcd_state == UFSHCD_STATE_RESET ||
6362                             hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
6363                         break;
6364                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6365                 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
6366                 flush_work(&hba->eh_work);
6367         } while (1);
6368
6369         hba->ufshcd_state = UFSHCD_STATE_RESET;
6370         ufshcd_set_eh_in_progress(hba);
6371         spin_unlock_irqrestore(hba->host->host_lock, flags);
6372
6373         err = ufshcd_reset_and_restore(hba);
6374
6375         spin_lock_irqsave(hba->host->host_lock, flags);
6376         if (!err) {
6377                 err = SUCCESS;
6378                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6379         } else {
6380                 err = FAILED;
6381                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6382         }
6383         ufshcd_clear_eh_in_progress(hba);
6384         spin_unlock_irqrestore(hba->host->host_lock, flags);
6385
6386         ufshcd_release(hba);
6387         return err;
6388 }
6389
6390 /**
6391  * ufshcd_get_max_icc_level - calculate the ICC level
6392  * @sup_curr_uA: max. current supported by the regulator
6393  * @start_scan: row at the desc table to start scan from
6394  * @buff: power descriptor buffer
6395  *
6396  * Returns calculated max ICC level for specific regulator
6397  */
6398 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6399 {
6400         int i;
6401         int curr_uA;
6402         u16 data;
6403         u16 unit;
6404
6405         for (i = start_scan; i >= 0; i--) {
6406                 data = be16_to_cpup((__be16 *)&buff[2 * i]);
6407                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6408                                                 ATTR_ICC_LVL_UNIT_OFFSET;
6409                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6410                 switch (unit) {
6411                 case UFSHCD_NANO_AMP:
6412                         curr_uA = curr_uA / 1000;
6413                         break;
6414                 case UFSHCD_MILI_AMP:
6415                         curr_uA = curr_uA * 1000;
6416                         break;
6417                 case UFSHCD_AMP:
6418                         curr_uA = curr_uA * 1000 * 1000;
6419                         break;
6420                 case UFSHCD_MICRO_AMP:
6421                 default:
6422                         break;
6423                 }
6424                 if (sup_curr_uA >= curr_uA)
6425                         break;
6426         }
6427         if (i < 0) {
6428                 i = 0;
6429                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6430         }
6431
6432         return (u32)i;
6433 }
6434
6435 /**
6436  * ufshcd_calc_icc_level - calculate the max ICC level
6437  * In case regulators are not initialized we'll return 0
6438  * @hba: per-adapter instance
6439  * @desc_buf: power descriptor buffer to extract ICC levels from.
6440  * @len: length of desc_buff
6441  *
6442  * Returns calculated ICC level
6443  */
6444 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6445                                                         u8 *desc_buf, int len)
6446 {
6447         u32 icc_level = 0;
6448
6449         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6450                                                 !hba->vreg_info.vccq2) {
6451                 dev_err(hba->dev,
6452                         "%s: Regulator capability was not set, actvIccLevel=%d",
6453                                                         __func__, icc_level);
6454                 goto out;
6455         }
6456
6457         if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
6458                 icc_level = ufshcd_get_max_icc_level(
6459                                 hba->vreg_info.vcc->max_uA,
6460                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6461                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6462
6463         if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
6464                 icc_level = ufshcd_get_max_icc_level(
6465                                 hba->vreg_info.vccq->max_uA,
6466                                 icc_level,
6467                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6468
6469         if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
6470                 icc_level = ufshcd_get_max_icc_level(
6471                                 hba->vreg_info.vccq2->max_uA,
6472                                 icc_level,
6473                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6474 out:
6475         return icc_level;
6476 }
6477
6478 static void ufshcd_init_icc_levels(struct ufs_hba *hba)
6479 {
6480         int ret;
6481         int buff_len = hba->desc_size.pwr_desc;
6482         u8 *desc_buf;
6483
6484         desc_buf = kmalloc(buff_len, GFP_KERNEL);
6485         if (!desc_buf)
6486                 return;
6487
6488         ret = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0,
6489                         desc_buf, buff_len);
6490         if (ret) {
6491                 dev_err(hba->dev,
6492                         "%s: Failed reading power descriptor.len = %d ret = %d",
6493                         __func__, buff_len, ret);
6494                 goto out;
6495         }
6496
6497         hba->init_prefetch_data.icc_level =
6498                         ufshcd_find_max_sup_active_icc_level(hba,
6499                         desc_buf, buff_len);
6500         dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
6501                         __func__, hba->init_prefetch_data.icc_level);
6502
6503         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6504                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
6505                 &hba->init_prefetch_data.icc_level);
6506
6507         if (ret)
6508                 dev_err(hba->dev,
6509                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
6510                         __func__, hba->init_prefetch_data.icc_level , ret);
6511
6512 out:
6513         kfree(desc_buf);
6514 }
6515
6516 /**
6517  * ufshcd_scsi_add_wlus - Adds required W-LUs
6518  * @hba: per-adapter instance
6519  *
6520  * UFS device specification requires the UFS devices to support 4 well known
6521  * logical units:
6522  *      "REPORT_LUNS" (address: 01h)
6523  *      "UFS Device" (address: 50h)
6524  *      "RPMB" (address: 44h)
6525  *      "BOOT" (address: 30h)
6526  * UFS device's power management needs to be controlled by "POWER CONDITION"
6527  * field of SSU (START STOP UNIT) command. But this "power condition" field
6528  * will take effect only when its sent to "UFS device" well known logical unit
6529  * hence we require the scsi_device instance to represent this logical unit in
6530  * order for the UFS host driver to send the SSU command for power management.
6531  *
6532  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6533  * Block) LU so user space process can control this LU. User space may also
6534  * want to have access to BOOT LU.
6535  *
6536  * This function adds scsi device instances for each of all well known LUs
6537  * (except "REPORT LUNS" LU).
6538  *
6539  * Returns zero on success (all required W-LUs are added successfully),
6540  * non-zero error value on failure (if failed to add any of the required W-LU).
6541  */
6542 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6543 {
6544         int ret = 0;
6545         struct scsi_device *sdev_rpmb;
6546         struct scsi_device *sdev_boot;
6547
6548         hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6549                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6550         if (IS_ERR(hba->sdev_ufs_device)) {
6551                 ret = PTR_ERR(hba->sdev_ufs_device);
6552                 hba->sdev_ufs_device = NULL;
6553                 goto out;
6554         }
6555         scsi_device_put(hba->sdev_ufs_device);
6556
6557         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
6558                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
6559         if (IS_ERR(sdev_rpmb)) {
6560                 ret = PTR_ERR(sdev_rpmb);
6561                 goto remove_sdev_ufs_device;
6562         }
6563         scsi_device_put(sdev_rpmb);
6564
6565         sdev_boot = __scsi_add_device(hba->host, 0, 0,
6566                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
6567         if (IS_ERR(sdev_boot))
6568                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
6569         else
6570                 scsi_device_put(sdev_boot);
6571         goto out;
6572
6573 remove_sdev_ufs_device:
6574         scsi_remove_device(hba->sdev_ufs_device);
6575 out:
6576         return ret;
6577 }
6578
6579 static int ufs_get_device_desc(struct ufs_hba *hba)
6580 {
6581         int err;
6582         size_t buff_len;
6583         u8 model_index;
6584         u8 *desc_buf;
6585         struct ufs_dev_info *dev_info = &hba->dev_info;
6586
6587         buff_len = max_t(size_t, hba->desc_size.dev_desc,
6588                          QUERY_DESC_MAX_SIZE + 1);
6589         desc_buf = kmalloc(buff_len, GFP_KERNEL);
6590         if (!desc_buf) {
6591                 err = -ENOMEM;
6592                 goto out;
6593         }
6594
6595         err = ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, desc_buf,
6596                         hba->desc_size.dev_desc);
6597         if (err) {
6598                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6599                         __func__, err);
6600                 goto out;
6601         }
6602
6603         /*
6604          * getting vendor (manufacturerID) and Bank Index in big endian
6605          * format
6606          */
6607         dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
6608                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6609
6610         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
6611         err = ufshcd_read_string_desc(hba, model_index,
6612                                       &dev_info->model, SD_ASCII_STD);
6613         if (err < 0) {
6614                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6615                         __func__, err);
6616                 goto out;
6617         }
6618
6619         /*
6620          * ufshcd_read_string_desc returns size of the string
6621          * reset the error value
6622          */
6623         err = 0;
6624
6625 out:
6626         kfree(desc_buf);
6627         return err;
6628 }
6629
6630 static void ufs_put_device_desc(struct ufs_hba *hba)
6631 {
6632         struct ufs_dev_info *dev_info = &hba->dev_info;
6633
6634         kfree(dev_info->model);
6635         dev_info->model = NULL;
6636 }
6637
6638 static void ufs_fixup_device_setup(struct ufs_hba *hba)
6639 {
6640         struct ufs_dev_fix *f;
6641         struct ufs_dev_info *dev_info = &hba->dev_info;
6642
6643         for (f = ufs_fixups; f->quirk; f++) {
6644                 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
6645                      f->wmanufacturerid == UFS_ANY_VENDOR) &&
6646                      ((dev_info->model &&
6647                        STR_PRFX_EQUAL(f->model, dev_info->model)) ||
6648                       !strcmp(f->model, UFS_ANY_MODEL)))
6649                         hba->dev_quirks |= f->quirk;
6650         }
6651 }
6652
6653 /**
6654  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6655  * @hba: per-adapter instance
6656  *
6657  * PA_TActivate parameter can be tuned manually if UniPro version is less than
6658  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6659  * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6660  * the hibern8 exit latency.
6661  *
6662  * Returns zero on success, non-zero error value on failure.
6663  */
6664 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6665 {
6666         int ret = 0;
6667         u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6668
6669         ret = ufshcd_dme_peer_get(hba,
6670                                   UIC_ARG_MIB_SEL(
6671                                         RX_MIN_ACTIVATETIME_CAPABILITY,
6672                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6673                                   &peer_rx_min_activatetime);
6674         if (ret)
6675                 goto out;
6676
6677         /* make sure proper unit conversion is applied */
6678         tuned_pa_tactivate =
6679                 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6680                  / PA_TACTIVATE_TIME_UNIT_US);
6681         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6682                              tuned_pa_tactivate);
6683
6684 out:
6685         return ret;
6686 }
6687
6688 /**
6689  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6690  * @hba: per-adapter instance
6691  *
6692  * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6693  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6694  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6695  * This optimal value can help reduce the hibern8 exit latency.
6696  *
6697  * Returns zero on success, non-zero error value on failure.
6698  */
6699 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6700 {
6701         int ret = 0;
6702         u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6703         u32 max_hibern8_time, tuned_pa_hibern8time;
6704
6705         ret = ufshcd_dme_get(hba,
6706                              UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6707                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6708                                   &local_tx_hibern8_time_cap);
6709         if (ret)
6710                 goto out;
6711
6712         ret = ufshcd_dme_peer_get(hba,
6713                                   UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6714                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6715                                   &peer_rx_hibern8_time_cap);
6716         if (ret)
6717                 goto out;
6718
6719         max_hibern8_time = max(local_tx_hibern8_time_cap,
6720                                peer_rx_hibern8_time_cap);
6721         /* make sure proper unit conversion is applied */
6722         tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6723                                 / PA_HIBERN8_TIME_UNIT_US);
6724         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6725                              tuned_pa_hibern8time);
6726 out:
6727         return ret;
6728 }
6729
6730 /**
6731  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6732  * less than device PA_TACTIVATE time.
6733  * @hba: per-adapter instance
6734  *
6735  * Some UFS devices require host PA_TACTIVATE to be lower than device
6736  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6737  * for such devices.
6738  *
6739  * Returns zero on success, non-zero error value on failure.
6740  */
6741 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6742 {
6743         int ret = 0;
6744         u32 granularity, peer_granularity;
6745         u32 pa_tactivate, peer_pa_tactivate;
6746         u32 pa_tactivate_us, peer_pa_tactivate_us;
6747         u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6748
6749         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6750                                   &granularity);
6751         if (ret)
6752                 goto out;
6753
6754         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6755                                   &peer_granularity);
6756         if (ret)
6757                 goto out;
6758
6759         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6760             (granularity > PA_GRANULARITY_MAX_VAL)) {
6761                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6762                         __func__, granularity);
6763                 return -EINVAL;
6764         }
6765
6766         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6767             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6768                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6769                         __func__, peer_granularity);
6770                 return -EINVAL;
6771         }
6772
6773         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6774         if (ret)
6775                 goto out;
6776
6777         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6778                                   &peer_pa_tactivate);
6779         if (ret)
6780                 goto out;
6781
6782         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6783         peer_pa_tactivate_us = peer_pa_tactivate *
6784                              gran_to_us_table[peer_granularity - 1];
6785
6786         if (pa_tactivate_us > peer_pa_tactivate_us) {
6787                 u32 new_peer_pa_tactivate;
6788
6789                 new_peer_pa_tactivate = pa_tactivate_us /
6790                                       gran_to_us_table[peer_granularity - 1];
6791                 new_peer_pa_tactivate++;
6792                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6793                                           new_peer_pa_tactivate);
6794         }
6795
6796 out:
6797         return ret;
6798 }
6799
6800 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
6801 {
6802         if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6803                 ufshcd_tune_pa_tactivate(hba);
6804                 ufshcd_tune_pa_hibern8time(hba);
6805         }
6806
6807         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6808                 /* set 1ms timeout for PA_TACTIVATE */
6809                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
6810
6811         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6812                 ufshcd_quirk_tune_host_pa_tactivate(hba);
6813
6814         ufshcd_vops_apply_dev_quirks(hba);
6815 }
6816
6817 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6818 {
6819         hba->ufs_stats.hibern8_exit_cnt = 0;
6820         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
6821         hba->req_abort_count = 0;
6822 }
6823
6824 static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6825 {
6826         int err;
6827
6828         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6829                 &hba->desc_size.dev_desc);
6830         if (err)
6831                 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6832
6833         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6834                 &hba->desc_size.pwr_desc);
6835         if (err)
6836                 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6837
6838         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6839                 &hba->desc_size.interc_desc);
6840         if (err)
6841                 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6842
6843         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6844                 &hba->desc_size.conf_desc);
6845         if (err)
6846                 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6847
6848         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6849                 &hba->desc_size.unit_desc);
6850         if (err)
6851                 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6852
6853         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6854                 &hba->desc_size.geom_desc);
6855         if (err)
6856                 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
6857
6858         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6859                 &hba->desc_size.hlth_desc);
6860         if (err)
6861                 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
6862 }
6863
6864 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
6865 {
6866         int err;
6867         size_t buff_len;
6868         u8 *desc_buf;
6869
6870         buff_len = hba->desc_size.geom_desc;
6871         desc_buf = kmalloc(buff_len, GFP_KERNEL);
6872         if (!desc_buf) {
6873                 err = -ENOMEM;
6874                 goto out;
6875         }
6876
6877         err = ufshcd_read_desc(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6878                         desc_buf, buff_len);
6879         if (err) {
6880                 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
6881                                 __func__, err);
6882                 goto out;
6883         }
6884
6885         if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
6886                 hba->dev_info.max_lu_supported = 32;
6887         else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
6888                 hba->dev_info.max_lu_supported = 8;
6889
6890 out:
6891         kfree(desc_buf);
6892         return err;
6893 }
6894
6895 static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
6896         {19200000, REF_CLK_FREQ_19_2_MHZ},
6897         {26000000, REF_CLK_FREQ_26_MHZ},
6898         {38400000, REF_CLK_FREQ_38_4_MHZ},
6899         {52000000, REF_CLK_FREQ_52_MHZ},
6900         {0, REF_CLK_FREQ_INVAL},
6901 };
6902
6903 static enum ufs_ref_clk_freq
6904 ufs_get_bref_clk_from_hz(unsigned long freq)
6905 {
6906         int i;
6907
6908         for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
6909                 if (ufs_ref_clk_freqs[i].freq_hz == freq)
6910                         return ufs_ref_clk_freqs[i].val;
6911
6912         return REF_CLK_FREQ_INVAL;
6913 }
6914
6915 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
6916 {
6917         unsigned long freq;
6918
6919         freq = clk_get_rate(refclk);
6920
6921         hba->dev_ref_clk_freq =
6922                 ufs_get_bref_clk_from_hz(freq);
6923
6924         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
6925                 dev_err(hba->dev,
6926                 "invalid ref_clk setting = %ld\n", freq);
6927 }
6928
6929 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
6930 {
6931         int err;
6932         u32 ref_clk;
6933         u32 freq = hba->dev_ref_clk_freq;
6934
6935         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6936                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
6937
6938         if (err) {
6939                 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
6940                         err);
6941                 goto out;
6942         }
6943
6944         if (ref_clk == freq)
6945                 goto out; /* nothing to update */
6946
6947         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6948                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
6949
6950         if (err) {
6951                 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
6952                         ufs_ref_clk_freqs[freq].freq_hz);
6953                 goto out;
6954         }
6955
6956         dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
6957                         ufs_ref_clk_freqs[freq].freq_hz);
6958
6959 out:
6960         return err;
6961 }
6962
6963 static int ufshcd_device_params_init(struct ufs_hba *hba)
6964 {
6965         bool flag;
6966         int ret;
6967
6968         /* Clear any previous UFS device information */
6969         memset(&hba->dev_info, 0, sizeof(hba->dev_info));
6970
6971         /* Init check for device descriptor sizes */
6972         ufshcd_init_desc_sizes(hba);
6973
6974         /* Init UFS geometry descriptor related parameters */
6975         ret = ufshcd_device_geo_params_init(hba);
6976         if (ret)
6977                 goto out;
6978
6979         /* Check and apply UFS device quirks */
6980         ret = ufs_get_device_desc(hba);
6981         if (ret) {
6982                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6983                         __func__, ret);
6984                 goto out;
6985         }
6986
6987         ufs_fixup_device_setup(hba);
6988
6989         if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
6990                         QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
6991                 hba->dev_info.f_power_on_wp_en = flag;
6992
6993         /* Probe maximum power mode co-supported by both UFS host and device */
6994         if (ufshcd_get_max_pwr_mode(hba))
6995                 dev_err(hba->dev,
6996                         "%s: Failed getting max supported power mode\n",
6997                         __func__);
6998 out:
6999         return ret;
7000 }
7001
7002 /**
7003  * ufshcd_add_lus - probe and add UFS logical units
7004  * @hba: per-adapter instance
7005  */
7006 static int ufshcd_add_lus(struct ufs_hba *hba)
7007 {
7008         int ret;
7009
7010         ufshcd_init_icc_levels(hba);
7011
7012         /* Add required well known logical units to scsi mid layer */
7013         ret = ufshcd_scsi_add_wlus(hba);
7014         if (ret)
7015                 goto out;
7016
7017         /* Initialize devfreq after UFS device is detected */
7018         if (ufshcd_is_clkscaling_supported(hba)) {
7019                 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7020                         &hba->pwr_info,
7021                         sizeof(struct ufs_pa_layer_attr));
7022                 hba->clk_scaling.saved_pwr_info.is_valid = true;
7023                 if (!hba->devfreq) {
7024                         ret = ufshcd_devfreq_init(hba);
7025                         if (ret)
7026                                 goto out;
7027                 }
7028
7029                 hba->clk_scaling.is_allowed = true;
7030         }
7031
7032         ufs_bsg_probe(hba);
7033         scsi_scan_host(hba->host);
7034         pm_runtime_put_sync(hba->dev);
7035
7036 out:
7037         return ret;
7038 }
7039
7040 /**
7041  * ufshcd_probe_hba - probe hba to detect device and initialize
7042  * @hba: per-adapter instance
7043  * @async: asynchronous execution or not
7044  *
7045  * Execute link-startup and verify device initialization
7046  */
7047 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
7048 {
7049         int ret;
7050         ktime_t start = ktime_get();
7051
7052         ret = ufshcd_link_startup(hba);
7053         if (ret)
7054                 goto out;
7055
7056         /* set the default level for urgent bkops */
7057         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
7058         hba->is_urgent_bkops_lvl_checked = false;
7059
7060         /* Debug counters initialization */
7061         ufshcd_clear_dbg_ufs_stats(hba);
7062
7063         /* UniPro link is active now */
7064         ufshcd_set_link_active(hba);
7065
7066         /* Verify device initialization by sending NOP OUT UPIU */
7067         ret = ufshcd_verify_dev_init(hba);
7068         if (ret)
7069                 goto out;
7070
7071         /* Initiate UFS initialization, and waiting until completion */
7072         ret = ufshcd_complete_dev_init(hba);
7073         if (ret)
7074                 goto out;
7075
7076         /*
7077          * Initialize UFS device parameters used by driver, these
7078          * parameters are associated with UFS descriptors.
7079          */
7080         if (async) {
7081                 ret = ufshcd_device_params_init(hba);
7082                 if (ret)
7083                         goto out;
7084         }
7085
7086         ufshcd_tune_unipro_params(hba);
7087
7088         /* UFS device is also active now */
7089         ufshcd_set_ufs_dev_active(hba);
7090         ufshcd_force_reset_auto_bkops(hba);
7091         hba->wlun_dev_clr_ua = true;
7092
7093         /* Gear up to HS gear if supported */
7094         if (hba->max_pwr_info.is_valid) {
7095                 /*
7096                  * Set the right value to bRefClkFreq before attempting to
7097                  * switch to HS gears.
7098                  */
7099                 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7100                         ufshcd_set_dev_ref_clk(hba);
7101                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
7102                 if (ret) {
7103                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7104                                         __func__, ret);
7105                         goto out;
7106                 }
7107         }
7108
7109         /* set the state as operational after switching to desired gear */
7110         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7111
7112         /* Enable Auto-Hibernate if configured */
7113         ufshcd_auto_hibern8_enable(hba);
7114
7115 out:
7116
7117         trace_ufshcd_init(dev_name(hba->dev), ret,
7118                 ktime_to_us(ktime_sub(ktime_get(), start)),
7119                 hba->curr_dev_pwr_mode, hba->uic_link_state);
7120         return ret;
7121 }
7122
7123 /**
7124  * ufshcd_async_scan - asynchronous execution for probing hba
7125  * @data: data pointer to pass to this function
7126  * @cookie: cookie data
7127  */
7128 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7129 {
7130         struct ufs_hba *hba = (struct ufs_hba *)data;
7131         int ret;
7132
7133         /* Initialize hba, detect and initialize UFS device */
7134         ret = ufshcd_probe_hba(hba, true);
7135         if (ret)
7136                 goto out;
7137
7138         /* Probe and add UFS logical units  */
7139         ret = ufshcd_add_lus(hba);
7140 out:
7141         /*
7142          * If we failed to initialize the device or the device is not
7143          * present, turn off the power/clocks etc.
7144          */
7145         if (ret) {
7146                 pm_runtime_put_sync(hba->dev);
7147                 ufshcd_exit_clk_scaling(hba);
7148                 ufshcd_hba_exit(hba);
7149         }
7150 }
7151
7152 static const struct attribute_group *ufshcd_driver_groups[] = {
7153         &ufs_sysfs_unit_descriptor_group,
7154         &ufs_sysfs_lun_attributes_group,
7155         NULL,
7156 };
7157
7158 static struct scsi_host_template ufshcd_driver_template = {
7159         .module                 = THIS_MODULE,
7160         .name                   = UFSHCD,
7161         .proc_name              = UFSHCD,
7162         .queuecommand           = ufshcd_queuecommand,
7163         .slave_alloc            = ufshcd_slave_alloc,
7164         .slave_configure        = ufshcd_slave_configure,
7165         .slave_destroy          = ufshcd_slave_destroy,
7166         .change_queue_depth     = ufshcd_change_queue_depth,
7167         .eh_abort_handler       = ufshcd_abort,
7168         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7169         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
7170         .this_id                = -1,
7171         .sg_tablesize           = SG_ALL,
7172         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
7173         .can_queue              = UFSHCD_CAN_QUEUE,
7174         .max_segment_size       = PRDT_DATA_BYTE_COUNT_MAX,
7175         .max_host_blocked       = 1,
7176         .track_queue_depth      = 1,
7177         .sdev_groups            = ufshcd_driver_groups,
7178         .dma_boundary           = PAGE_SIZE - 1,
7179         .rpm_autosuspend_delay  = RPM_AUTOSUSPEND_DELAY_MS,
7180 };
7181
7182 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7183                                    int ua)
7184 {
7185         int ret;
7186
7187         if (!vreg)
7188                 return 0;
7189
7190         /*
7191          * "set_load" operation shall be required on those regulators
7192          * which specifically configured current limitation. Otherwise
7193          * zero max_uA may cause unexpected behavior when regulator is
7194          * enabled or set as high power mode.
7195          */
7196         if (!vreg->max_uA)
7197                 return 0;
7198
7199         ret = regulator_set_load(vreg->reg, ua);
7200         if (ret < 0) {
7201                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7202                                 __func__, vreg->name, ua, ret);
7203         }
7204
7205         return ret;
7206 }
7207
7208 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7209                                          struct ufs_vreg *vreg)
7210 {
7211         return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
7212 }
7213
7214 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7215                                          struct ufs_vreg *vreg)
7216 {
7217         if (!vreg)
7218                 return 0;
7219
7220         return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
7221 }
7222
7223 static int ufshcd_config_vreg(struct device *dev,
7224                 struct ufs_vreg *vreg, bool on)
7225 {
7226         int ret = 0;
7227         struct regulator *reg;
7228         const char *name;
7229         int min_uV, uA_load;
7230
7231         BUG_ON(!vreg);
7232
7233         reg = vreg->reg;
7234         name = vreg->name;
7235
7236         if (regulator_count_voltages(reg) > 0) {
7237                 if (vreg->min_uV && vreg->max_uV) {
7238                         min_uV = on ? vreg->min_uV : 0;
7239                         ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7240                         if (ret) {
7241                                 dev_err(dev,
7242                                         "%s: %s set voltage failed, err=%d\n",
7243                                         __func__, name, ret);
7244                                 goto out;
7245                         }
7246                 }
7247
7248                 uA_load = on ? vreg->max_uA : 0;
7249                 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7250                 if (ret)
7251                         goto out;
7252         }
7253 out:
7254         return ret;
7255 }
7256
7257 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7258 {
7259         int ret = 0;
7260
7261         if (!vreg || vreg->enabled)
7262                 goto out;
7263
7264         ret = ufshcd_config_vreg(dev, vreg, true);
7265         if (!ret)
7266                 ret = regulator_enable(vreg->reg);
7267
7268         if (!ret)
7269                 vreg->enabled = true;
7270         else
7271                 dev_err(dev, "%s: %s enable failed, err=%d\n",
7272                                 __func__, vreg->name, ret);
7273 out:
7274         return ret;
7275 }
7276
7277 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7278 {
7279         int ret = 0;
7280
7281         if (!vreg || !vreg->enabled)
7282                 goto out;
7283
7284         ret = regulator_disable(vreg->reg);
7285
7286         if (!ret) {
7287                 /* ignore errors on applying disable config */
7288                 ufshcd_config_vreg(dev, vreg, false);
7289                 vreg->enabled = false;
7290         } else {
7291                 dev_err(dev, "%s: %s disable failed, err=%d\n",
7292                                 __func__, vreg->name, ret);
7293         }
7294 out:
7295         return ret;
7296 }
7297
7298 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7299 {
7300         int ret = 0;
7301         struct device *dev = hba->dev;
7302         struct ufs_vreg_info *info = &hba->vreg_info;
7303
7304         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7305         if (ret)
7306                 goto out;
7307
7308         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7309         if (ret)
7310                 goto out;
7311
7312         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7313         if (ret)
7314                 goto out;
7315
7316 out:
7317         if (ret) {
7318                 ufshcd_toggle_vreg(dev, info->vccq2, false);
7319                 ufshcd_toggle_vreg(dev, info->vccq, false);
7320                 ufshcd_toggle_vreg(dev, info->vcc, false);
7321         }
7322         return ret;
7323 }
7324
7325 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7326 {
7327         struct ufs_vreg_info *info = &hba->vreg_info;
7328
7329         return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
7330 }
7331
7332 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7333 {
7334         int ret = 0;
7335
7336         if (!vreg)
7337                 goto out;
7338
7339         vreg->reg = devm_regulator_get(dev, vreg->name);
7340         if (IS_ERR(vreg->reg)) {
7341                 ret = PTR_ERR(vreg->reg);
7342                 dev_err(dev, "%s: %s get failed, err=%d\n",
7343                                 __func__, vreg->name, ret);
7344         }
7345 out:
7346         return ret;
7347 }
7348
7349 static int ufshcd_init_vreg(struct ufs_hba *hba)
7350 {
7351         int ret = 0;
7352         struct device *dev = hba->dev;
7353         struct ufs_vreg_info *info = &hba->vreg_info;
7354
7355         ret = ufshcd_get_vreg(dev, info->vcc);
7356         if (ret)
7357                 goto out;
7358
7359         ret = ufshcd_get_vreg(dev, info->vccq);
7360         if (ret)
7361                 goto out;
7362
7363         ret = ufshcd_get_vreg(dev, info->vccq2);
7364 out:
7365         return ret;
7366 }
7367
7368 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7369 {
7370         struct ufs_vreg_info *info = &hba->vreg_info;
7371
7372         if (info)
7373                 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7374
7375         return 0;
7376 }
7377
7378 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7379                                         bool skip_ref_clk)
7380 {
7381         int ret = 0;
7382         struct ufs_clk_info *clki;
7383         struct list_head *head = &hba->clk_list_head;
7384         unsigned long flags;
7385         ktime_t start = ktime_get();
7386         bool clk_state_changed = false;
7387
7388         if (list_empty(head))
7389                 goto out;
7390
7391         /*
7392          * vendor specific setup_clocks ops may depend on clocks managed by
7393          * this standard driver hence call the vendor specific setup_clocks
7394          * before disabling the clocks managed here.
7395          */
7396         if (!on) {
7397                 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7398                 if (ret)
7399                         return ret;
7400         }
7401
7402         list_for_each_entry(clki, head, list) {
7403                 if (!IS_ERR_OR_NULL(clki->clk)) {
7404                         if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7405                                 continue;
7406
7407                         clk_state_changed = on ^ clki->enabled;
7408                         if (on && !clki->enabled) {
7409                                 ret = clk_prepare_enable(clki->clk);
7410                                 if (ret) {
7411                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7412                                                 __func__, clki->name, ret);
7413                                         goto out;
7414                                 }
7415                         } else if (!on && clki->enabled) {
7416                                 clk_disable_unprepare(clki->clk);
7417                         }
7418                         clki->enabled = on;
7419                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7420                                         clki->name, on ? "en" : "dis");
7421                 }
7422         }
7423
7424         /*
7425          * vendor specific setup_clocks ops may depend on clocks managed by
7426          * this standard driver hence call the vendor specific setup_clocks
7427          * after enabling the clocks managed here.
7428          */
7429         if (on) {
7430                 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7431                 if (ret)
7432                         return ret;
7433         }
7434
7435 out:
7436         if (ret) {
7437                 list_for_each_entry(clki, head, list) {
7438                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7439                                 clk_disable_unprepare(clki->clk);
7440                 }
7441         } else if (!ret && on) {
7442                 spin_lock_irqsave(hba->host->host_lock, flags);
7443                 hba->clk_gating.state = CLKS_ON;
7444                 trace_ufshcd_clk_gating(dev_name(hba->dev),
7445                                         hba->clk_gating.state);
7446                 spin_unlock_irqrestore(hba->host->host_lock, flags);
7447         }
7448
7449         if (clk_state_changed)
7450                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7451                         (on ? "on" : "off"),
7452                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
7453         return ret;
7454 }
7455
7456 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7457 {
7458         return  __ufshcd_setup_clocks(hba, on, false);
7459 }
7460
7461 static int ufshcd_init_clocks(struct ufs_hba *hba)
7462 {
7463         int ret = 0;
7464         struct ufs_clk_info *clki;
7465         struct device *dev = hba->dev;
7466         struct list_head *head = &hba->clk_list_head;
7467
7468         if (list_empty(head))
7469                 goto out;
7470
7471         list_for_each_entry(clki, head, list) {
7472                 if (!clki->name)
7473                         continue;
7474
7475                 clki->clk = devm_clk_get(dev, clki->name);
7476                 if (IS_ERR(clki->clk)) {
7477                         ret = PTR_ERR(clki->clk);
7478                         dev_err(dev, "%s: %s clk get failed, %d\n",
7479                                         __func__, clki->name, ret);
7480                         goto out;
7481                 }
7482
7483                 /*
7484                  * Parse device ref clk freq as per device tree "ref_clk".
7485                  * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7486                  * in ufshcd_alloc_host().
7487                  */
7488                 if (!strcmp(clki->name, "ref_clk"))
7489                         ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
7490
7491                 if (clki->max_freq) {
7492                         ret = clk_set_rate(clki->clk, clki->max_freq);
7493                         if (ret) {
7494                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7495                                         __func__, clki->name,
7496                                         clki->max_freq, ret);
7497                                 goto out;
7498                         }
7499                         clki->curr_freq = clki->max_freq;
7500                 }
7501                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7502                                 clki->name, clk_get_rate(clki->clk));
7503         }
7504 out:
7505         return ret;
7506 }
7507
7508 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7509 {
7510         int err = 0;
7511
7512         if (!hba->vops)
7513                 goto out;
7514
7515         err = ufshcd_vops_init(hba);
7516         if (err)
7517                 goto out;
7518
7519         err = ufshcd_vops_setup_regulators(hba, true);
7520         if (err)
7521                 goto out_exit;
7522
7523         goto out;
7524
7525 out_exit:
7526         ufshcd_vops_exit(hba);
7527 out:
7528         if (err)
7529                 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
7530                         __func__, ufshcd_get_var_name(hba), err);
7531         return err;
7532 }
7533
7534 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7535 {
7536         if (!hba->vops)
7537                 return;
7538
7539         ufshcd_vops_setup_regulators(hba, false);
7540
7541         ufshcd_vops_exit(hba);
7542 }
7543
7544 static int ufshcd_hba_init(struct ufs_hba *hba)
7545 {
7546         int err;
7547
7548         /*
7549          * Handle host controller power separately from the UFS device power
7550          * rails as it will help controlling the UFS host controller power
7551          * collapse easily which is different than UFS device power collapse.
7552          * Also, enable the host controller power before we go ahead with rest
7553          * of the initialization here.
7554          */
7555         err = ufshcd_init_hba_vreg(hba);
7556         if (err)
7557                 goto out;
7558
7559         err = ufshcd_setup_hba_vreg(hba, true);
7560         if (err)
7561                 goto out;
7562
7563         err = ufshcd_init_clocks(hba);
7564         if (err)
7565                 goto out_disable_hba_vreg;
7566
7567         err = ufshcd_setup_clocks(hba, true);
7568         if (err)
7569                 goto out_disable_hba_vreg;
7570
7571         err = ufshcd_init_vreg(hba);
7572         if (err)
7573                 goto out_disable_clks;
7574
7575         err = ufshcd_setup_vreg(hba, true);
7576         if (err)
7577                 goto out_disable_clks;
7578
7579         err = ufshcd_variant_hba_init(hba);
7580         if (err)
7581                 goto out_disable_vreg;
7582
7583         hba->is_powered = true;
7584         goto out;
7585
7586 out_disable_vreg:
7587         ufshcd_setup_vreg(hba, false);
7588 out_disable_clks:
7589         ufshcd_setup_clocks(hba, false);
7590 out_disable_hba_vreg:
7591         ufshcd_setup_hba_vreg(hba, false);
7592 out:
7593         return err;
7594 }
7595
7596 static void ufshcd_hba_exit(struct ufs_hba *hba)
7597 {
7598         if (hba->is_powered) {
7599                 ufshcd_variant_hba_exit(hba);
7600                 ufshcd_setup_vreg(hba, false);
7601                 ufshcd_suspend_clkscaling(hba);
7602                 if (ufshcd_is_clkscaling_supported(hba))
7603                         if (hba->devfreq)
7604                                 ufshcd_suspend_clkscaling(hba);
7605                 ufshcd_setup_clocks(hba, false);
7606                 ufshcd_setup_hba_vreg(hba, false);
7607                 hba->is_powered = false;
7608                 ufs_put_device_desc(hba);
7609         }
7610 }
7611
7612 static int
7613 ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7614 {
7615         unsigned char cmd[6] = {REQUEST_SENSE,
7616                                 0,
7617                                 0,
7618                                 0,
7619                                 UFS_SENSE_SIZE,
7620                                 0};
7621         char *buffer;
7622         int ret;
7623
7624         buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
7625         if (!buffer) {
7626                 ret = -ENOMEM;
7627                 goto out;
7628         }
7629
7630         ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
7631                         UFS_SENSE_SIZE, NULL, NULL,
7632                         msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
7633         if (ret)
7634                 pr_err("%s: failed with err %d\n", __func__, ret);
7635
7636         kfree(buffer);
7637 out:
7638         return ret;
7639 }
7640
7641 /**
7642  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7643  *                           power mode
7644  * @hba: per adapter instance
7645  * @pwr_mode: device power mode to set
7646  *
7647  * Returns 0 if requested power mode is set successfully
7648  * Returns non-zero if failed to set the requested power mode
7649  */
7650 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7651                                      enum ufs_dev_pwr_mode pwr_mode)
7652 {
7653         unsigned char cmd[6] = { START_STOP };
7654         struct scsi_sense_hdr sshdr;
7655         struct scsi_device *sdp;
7656         unsigned long flags;
7657         int ret;
7658
7659         spin_lock_irqsave(hba->host->host_lock, flags);
7660         sdp = hba->sdev_ufs_device;
7661         if (sdp) {
7662                 ret = scsi_device_get(sdp);
7663                 if (!ret && !scsi_device_online(sdp)) {
7664                         ret = -ENODEV;
7665                         scsi_device_put(sdp);
7666                 }
7667         } else {
7668                 ret = -ENODEV;
7669         }
7670         spin_unlock_irqrestore(hba->host->host_lock, flags);
7671
7672         if (ret)
7673                 return ret;
7674
7675         /*
7676          * If scsi commands fail, the scsi mid-layer schedules scsi error-
7677          * handling, which would wait for host to be resumed. Since we know
7678          * we are functional while we are here, skip host resume in error
7679          * handling context.
7680          */
7681         hba->host->eh_noresume = 1;
7682         if (hba->wlun_dev_clr_ua) {
7683                 ret = ufshcd_send_request_sense(hba, sdp);
7684                 if (ret)
7685                         goto out;
7686                 /* Unit attention condition is cleared now */
7687                 hba->wlun_dev_clr_ua = false;
7688         }
7689
7690         cmd[4] = pwr_mode << 4;
7691
7692         /*
7693          * Current function would be generally called from the power management
7694          * callbacks hence set the RQF_PM flag so that it doesn't resume the
7695          * already suspended childs.
7696          */
7697         ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7698                         START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
7699         if (ret) {
7700                 sdev_printk(KERN_WARNING, sdp,
7701                             "START_STOP failed for power mode: %d, result %x\n",
7702                             pwr_mode, ret);
7703                 if (driver_byte(ret) == DRIVER_SENSE)
7704                         scsi_print_sense_hdr(sdp, NULL, &sshdr);
7705         }
7706
7707         if (!ret)
7708                 hba->curr_dev_pwr_mode = pwr_mode;
7709 out:
7710         scsi_device_put(sdp);
7711         hba->host->eh_noresume = 0;
7712         return ret;
7713 }
7714
7715 static int ufshcd_link_state_transition(struct ufs_hba *hba,
7716                                         enum uic_link_state req_link_state,
7717                                         int check_for_bkops)
7718 {
7719         int ret = 0;
7720
7721         if (req_link_state == hba->uic_link_state)
7722                 return 0;
7723
7724         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7725                 ret = ufshcd_uic_hibern8_enter(hba);
7726                 if (!ret)
7727                         ufshcd_set_link_hibern8(hba);
7728                 else
7729                         goto out;
7730         }
7731         /*
7732          * If autobkops is enabled, link can't be turned off because
7733          * turning off the link would also turn off the device.
7734          */
7735         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
7736                  (!check_for_bkops || !hba->auto_bkops_enabled)) {
7737                 /*
7738                  * Let's make sure that link is in low power mode, we are doing
7739                  * this currently by putting the link in Hibern8. Otherway to
7740                  * put the link in low power mode is to send the DME end point
7741                  * to device and then send the DME reset command to local
7742                  * unipro. But putting the link in hibern8 is much faster.
7743                  */
7744                 ret = ufshcd_uic_hibern8_enter(hba);
7745                 if (ret)
7746                         goto out;
7747                 /*
7748                  * Change controller state to "reset state" which
7749                  * should also put the link in off/reset state
7750                  */
7751                 ufshcd_hba_stop(hba, true);
7752                 /*
7753                  * TODO: Check if we need any delay to make sure that
7754                  * controller is reset
7755                  */
7756                 ufshcd_set_link_off(hba);
7757         }
7758
7759 out:
7760         return ret;
7761 }
7762
7763 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7764 {
7765         /*
7766          * It seems some UFS devices may keep drawing more than sleep current
7767          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7768          * To avoid this situation, add 2ms delay before putting these UFS
7769          * rails in LPM mode.
7770          */
7771         if (!ufshcd_is_link_active(hba) &&
7772             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7773                 usleep_range(2000, 2100);
7774
7775         /*
7776          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7777          * power.
7778          *
7779          * If UFS device and link is in OFF state, all power supplies (VCC,
7780          * VCCQ, VCCQ2) can be turned off if power on write protect is not
7781          * required. If UFS link is inactive (Hibern8 or OFF state) and device
7782          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7783          *
7784          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7785          * in low power state which would save some power.
7786          */
7787         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7788             !hba->dev_info.is_lu_power_on_wp) {
7789                 ufshcd_setup_vreg(hba, false);
7790         } else if (!ufshcd_is_ufs_dev_active(hba)) {
7791                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7792                 if (!ufshcd_is_link_active(hba)) {
7793                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7794                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7795                 }
7796         }
7797 }
7798
7799 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7800 {
7801         int ret = 0;
7802
7803         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7804             !hba->dev_info.is_lu_power_on_wp) {
7805                 ret = ufshcd_setup_vreg(hba, true);
7806         } else if (!ufshcd_is_ufs_dev_active(hba)) {
7807                 if (!ret && !ufshcd_is_link_active(hba)) {
7808                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7809                         if (ret)
7810                                 goto vcc_disable;
7811                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7812                         if (ret)
7813                                 goto vccq_lpm;
7814                 }
7815                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
7816         }
7817         goto out;
7818
7819 vccq_lpm:
7820         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7821 vcc_disable:
7822         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7823 out:
7824         return ret;
7825 }
7826
7827 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7828 {
7829         if (ufshcd_is_link_off(hba))
7830                 ufshcd_setup_hba_vreg(hba, false);
7831 }
7832
7833 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7834 {
7835         if (ufshcd_is_link_off(hba))
7836                 ufshcd_setup_hba_vreg(hba, true);
7837 }
7838
7839 /**
7840  * ufshcd_suspend - helper function for suspend operations
7841  * @hba: per adapter instance
7842  * @pm_op: desired low power operation type
7843  *
7844  * This function will try to put the UFS device and link into low power
7845  * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7846  * (System PM level).
7847  *
7848  * If this function is called during shutdown, it will make sure that
7849  * both UFS device and UFS link is powered off.
7850  *
7851  * NOTE: UFS device & link must be active before we enter in this function.
7852  *
7853  * Returns 0 for success and non-zero for failure
7854  */
7855 static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7856 {
7857         int ret = 0;
7858         enum ufs_pm_level pm_lvl;
7859         enum ufs_dev_pwr_mode req_dev_pwr_mode;
7860         enum uic_link_state req_link_state;
7861
7862         hba->pm_op_in_progress = 1;
7863         if (!ufshcd_is_shutdown_pm(pm_op)) {
7864                 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7865                          hba->rpm_lvl : hba->spm_lvl;
7866                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7867                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7868         } else {
7869                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7870                 req_link_state = UIC_LINK_OFF_STATE;
7871         }
7872
7873         /*
7874          * If we can't transition into any of the low power modes
7875          * just gate the clocks.
7876          */
7877         ufshcd_hold(hba, false);
7878         hba->clk_gating.is_suspended = true;
7879
7880         if (hba->clk_scaling.is_allowed) {
7881                 cancel_work_sync(&hba->clk_scaling.suspend_work);
7882                 cancel_work_sync(&hba->clk_scaling.resume_work);
7883                 ufshcd_suspend_clkscaling(hba);
7884         }
7885
7886         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7887                         req_link_state == UIC_LINK_ACTIVE_STATE) {
7888                 goto disable_clks;
7889         }
7890
7891         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7892             (req_link_state == hba->uic_link_state))
7893                 goto enable_gating;
7894
7895         /* UFS device & link must be active before we enter in this function */
7896         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7897                 ret = -EINVAL;
7898                 goto enable_gating;
7899         }
7900
7901         if (ufshcd_is_runtime_pm(pm_op)) {
7902                 if (ufshcd_can_autobkops_during_suspend(hba)) {
7903                         /*
7904                          * The device is idle with no requests in the queue,
7905                          * allow background operations if bkops status shows
7906                          * that performance might be impacted.
7907                          */
7908                         ret = ufshcd_urgent_bkops(hba);
7909                         if (ret)
7910                                 goto enable_gating;
7911                 } else {
7912                         /* make sure that auto bkops is disabled */
7913                         ufshcd_disable_auto_bkops(hba);
7914                 }
7915         }
7916
7917         if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7918              ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7919                !ufshcd_is_runtime_pm(pm_op))) {
7920                 /* ensure that bkops is disabled */
7921                 ufshcd_disable_auto_bkops(hba);
7922                 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7923                 if (ret)
7924                         goto enable_gating;
7925         }
7926
7927         ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7928         if (ret)
7929                 goto set_dev_active;
7930
7931         ufshcd_vreg_set_lpm(hba);
7932
7933 disable_clks:
7934         /*
7935          * Call vendor specific suspend callback. As these callbacks may access
7936          * vendor specific host controller register space call them before the
7937          * host clocks are ON.
7938          */
7939         ret = ufshcd_vops_suspend(hba, pm_op);
7940         if (ret)
7941                 goto set_link_active;
7942         /*
7943          * Disable the host irq as host controller as there won't be any
7944          * host controller transaction expected till resume.
7945          */
7946         ufshcd_disable_irq(hba);
7947
7948         if (!ufshcd_is_link_active(hba))
7949                 ufshcd_setup_clocks(hba, false);
7950         else
7951                 /* If link is active, device ref_clk can't be switched off */
7952                 __ufshcd_setup_clocks(hba, false, true);
7953
7954         hba->clk_gating.state = CLKS_OFF;
7955         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
7956
7957         /* Put the host controller in low power mode if possible */
7958         ufshcd_hba_vreg_set_lpm(hba);
7959         goto out;
7960
7961 set_link_active:
7962         if (hba->clk_scaling.is_allowed)
7963                 ufshcd_resume_clkscaling(hba);
7964         ufshcd_vreg_set_hpm(hba);
7965         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7966                 ufshcd_set_link_active(hba);
7967         else if (ufshcd_is_link_off(hba))
7968                 ufshcd_host_reset_and_restore(hba);
7969 set_dev_active:
7970         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7971                 ufshcd_disable_auto_bkops(hba);
7972 enable_gating:
7973         if (hba->clk_scaling.is_allowed)
7974                 ufshcd_resume_clkscaling(hba);
7975         hba->clk_gating.is_suspended = false;
7976         ufshcd_release(hba);
7977 out:
7978         hba->pm_op_in_progress = 0;
7979         if (ret)
7980                 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
7981         return ret;
7982 }
7983
7984 /**
7985  * ufshcd_resume - helper function for resume operations
7986  * @hba: per adapter instance
7987  * @pm_op: runtime PM or system PM
7988  *
7989  * This function basically brings the UFS device, UniPro link and controller
7990  * to active state.
7991  *
7992  * Returns 0 for success and non-zero for failure
7993  */
7994 static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7995 {
7996         int ret;
7997         enum uic_link_state old_link_state;
7998
7999         hba->pm_op_in_progress = 1;
8000         old_link_state = hba->uic_link_state;
8001
8002         ufshcd_hba_vreg_set_hpm(hba);
8003         /* Make sure clocks are enabled before accessing controller */
8004         ret = ufshcd_setup_clocks(hba, true);
8005         if (ret)
8006                 goto out;
8007
8008         /* enable the host irq as host controller would be active soon */
8009         ufshcd_enable_irq(hba);
8010
8011         ret = ufshcd_vreg_set_hpm(hba);
8012         if (ret)
8013                 goto disable_irq_and_vops_clks;
8014
8015         /*
8016          * Call vendor specific resume callback. As these callbacks may access
8017          * vendor specific host controller register space call them when the
8018          * host clocks are ON.
8019          */
8020         ret = ufshcd_vops_resume(hba, pm_op);
8021         if (ret)
8022                 goto disable_vreg;
8023
8024         if (ufshcd_is_link_hibern8(hba)) {
8025                 ret = ufshcd_uic_hibern8_exit(hba);
8026                 if (!ret)
8027                         ufshcd_set_link_active(hba);
8028                 else
8029                         goto vendor_suspend;
8030         } else if (ufshcd_is_link_off(hba)) {
8031                 ret = ufshcd_host_reset_and_restore(hba);
8032                 /*
8033                  * ufshcd_host_reset_and_restore() should have already
8034                  * set the link state as active
8035                  */
8036                 if (ret || !ufshcd_is_link_active(hba))
8037                         goto vendor_suspend;
8038         }
8039
8040         if (!ufshcd_is_ufs_dev_active(hba)) {
8041                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8042                 if (ret)
8043                         goto set_old_link_state;
8044         }
8045
8046         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8047                 ufshcd_enable_auto_bkops(hba);
8048         else
8049                 /*
8050                  * If BKOPs operations are urgently needed at this moment then
8051                  * keep auto-bkops enabled or else disable it.
8052                  */
8053                 ufshcd_urgent_bkops(hba);
8054
8055         hba->clk_gating.is_suspended = false;
8056
8057         if (hba->clk_scaling.is_allowed)
8058                 ufshcd_resume_clkscaling(hba);
8059
8060         /* Enable Auto-Hibernate if configured */
8061         ufshcd_auto_hibern8_enable(hba);
8062
8063         /* Schedule clock gating in case of no access to UFS device yet */
8064         ufshcd_release(hba);
8065
8066         goto out;
8067
8068 set_old_link_state:
8069         ufshcd_link_state_transition(hba, old_link_state, 0);
8070 vendor_suspend:
8071         ufshcd_vops_suspend(hba, pm_op);
8072 disable_vreg:
8073         ufshcd_vreg_set_lpm(hba);
8074 disable_irq_and_vops_clks:
8075         ufshcd_disable_irq(hba);
8076         if (hba->clk_scaling.is_allowed)
8077                 ufshcd_suspend_clkscaling(hba);
8078         ufshcd_setup_clocks(hba, false);
8079 out:
8080         hba->pm_op_in_progress = 0;
8081         if (ret)
8082                 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
8083         return ret;
8084 }
8085
8086 /**
8087  * ufshcd_system_suspend - system suspend routine
8088  * @hba: per adapter instance
8089  *
8090  * Check the description of ufshcd_suspend() function for more details.
8091  *
8092  * Returns 0 for success and non-zero for failure
8093  */
8094 int ufshcd_system_suspend(struct ufs_hba *hba)
8095 {
8096         int ret = 0;
8097         ktime_t start = ktime_get();
8098
8099         if (!hba || !hba->is_powered)
8100                 return 0;
8101
8102         if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8103              hba->curr_dev_pwr_mode) &&
8104             (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8105              hba->uic_link_state))
8106                 goto out;
8107
8108         if (pm_runtime_suspended(hba->dev)) {
8109                 /*
8110                  * UFS device and/or UFS link low power states during runtime
8111                  * suspend seems to be different than what is expected during
8112                  * system suspend. Hence runtime resume the devic & link and
8113                  * let the system suspend low power states to take effect.
8114                  * TODO: If resume takes longer time, we might have optimize
8115                  * it in future by not resuming everything if possible.
8116                  */
8117                 ret = ufshcd_runtime_resume(hba);
8118                 if (ret)
8119                         goto out;
8120         }
8121
8122         ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8123 out:
8124         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8125                 ktime_to_us(ktime_sub(ktime_get(), start)),
8126                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8127         if (!ret)
8128                 hba->is_sys_suspended = true;
8129         return ret;
8130 }
8131 EXPORT_SYMBOL(ufshcd_system_suspend);
8132
8133 /**
8134  * ufshcd_system_resume - system resume routine
8135  * @hba: per adapter instance
8136  *
8137  * Returns 0 for success and non-zero for failure
8138  */
8139
8140 int ufshcd_system_resume(struct ufs_hba *hba)
8141 {
8142         int ret = 0;
8143         ktime_t start = ktime_get();
8144
8145         if (!hba)
8146                 return -EINVAL;
8147
8148         if (!hba->is_powered || pm_runtime_suspended(hba->dev))
8149                 /*
8150                  * Let the runtime resume take care of resuming
8151                  * if runtime suspended.
8152                  */
8153                 goto out;
8154         else
8155                 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8156 out:
8157         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8158                 ktime_to_us(ktime_sub(ktime_get(), start)),
8159                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8160         if (!ret)
8161                 hba->is_sys_suspended = false;
8162         return ret;
8163 }
8164 EXPORT_SYMBOL(ufshcd_system_resume);
8165
8166 /**
8167  * ufshcd_runtime_suspend - runtime suspend routine
8168  * @hba: per adapter instance
8169  *
8170  * Check the description of ufshcd_suspend() function for more details.
8171  *
8172  * Returns 0 for success and non-zero for failure
8173  */
8174 int ufshcd_runtime_suspend(struct ufs_hba *hba)
8175 {
8176         int ret = 0;
8177         ktime_t start = ktime_get();
8178
8179         if (!hba)
8180                 return -EINVAL;
8181
8182         if (!hba->is_powered)
8183                 goto out;
8184         else
8185                 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8186 out:
8187         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8188                 ktime_to_us(ktime_sub(ktime_get(), start)),
8189                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8190         return ret;
8191 }
8192 EXPORT_SYMBOL(ufshcd_runtime_suspend);
8193
8194 /**
8195  * ufshcd_runtime_resume - runtime resume routine
8196  * @hba: per adapter instance
8197  *
8198  * This function basically brings the UFS device, UniPro link and controller
8199  * to active state. Following operations are done in this function:
8200  *
8201  * 1. Turn on all the controller related clocks
8202  * 2. Bring the UniPro link out of Hibernate state
8203  * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8204  *    to active state.
8205  * 4. If auto-bkops is enabled on the device, disable it.
8206  *
8207  * So following would be the possible power state after this function return
8208  * successfully:
8209  *      S1: UFS device in Active state with VCC rail ON
8210  *          UniPro link in Active state
8211  *          All the UFS/UniPro controller clocks are ON
8212  *
8213  * Returns 0 for success and non-zero for failure
8214  */
8215 int ufshcd_runtime_resume(struct ufs_hba *hba)
8216 {
8217         int ret = 0;
8218         ktime_t start = ktime_get();
8219
8220         if (!hba)
8221                 return -EINVAL;
8222
8223         if (!hba->is_powered)
8224                 goto out;
8225         else
8226                 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8227 out:
8228         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8229                 ktime_to_us(ktime_sub(ktime_get(), start)),
8230                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8231         return ret;
8232 }
8233 EXPORT_SYMBOL(ufshcd_runtime_resume);
8234
8235 int ufshcd_runtime_idle(struct ufs_hba *hba)
8236 {
8237         return 0;
8238 }
8239 EXPORT_SYMBOL(ufshcd_runtime_idle);
8240
8241 /**
8242  * ufshcd_shutdown - shutdown routine
8243  * @hba: per adapter instance
8244  *
8245  * This function would power off both UFS device and UFS link.
8246  *
8247  * Returns 0 always to allow force shutdown even in case of errors.
8248  */
8249 int ufshcd_shutdown(struct ufs_hba *hba)
8250 {
8251         int ret = 0;
8252
8253         if (!hba->is_powered)
8254                 goto out;
8255
8256         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8257                 goto out;
8258
8259         if (pm_runtime_suspended(hba->dev)) {
8260                 ret = ufshcd_runtime_resume(hba);
8261                 if (ret)
8262                         goto out;
8263         }
8264
8265         ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8266 out:
8267         if (ret)
8268                 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8269         /* allow force shutdown even in case of errors */
8270         return 0;
8271 }
8272 EXPORT_SYMBOL(ufshcd_shutdown);
8273
8274 /**
8275  * ufshcd_remove - de-allocate SCSI host and host memory space
8276  *              data structure memory
8277  * @hba: per adapter instance
8278  */
8279 void ufshcd_remove(struct ufs_hba *hba)
8280 {
8281         ufs_bsg_remove(hba);
8282         ufs_sysfs_remove_nodes(hba->dev);
8283         blk_cleanup_queue(hba->tmf_queue);
8284         blk_mq_free_tag_set(&hba->tmf_tag_set);
8285         blk_cleanup_queue(hba->cmd_queue);
8286         scsi_remove_host(hba->host);
8287         /* disable interrupts */
8288         ufshcd_disable_intr(hba, hba->intr_mask);
8289         ufshcd_hba_stop(hba, true);
8290
8291         ufshcd_exit_clk_scaling(hba);
8292         ufshcd_exit_clk_gating(hba);
8293         if (ufshcd_is_clkscaling_supported(hba))
8294                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
8295         ufshcd_hba_exit(hba);
8296 }
8297 EXPORT_SYMBOL_GPL(ufshcd_remove);
8298
8299 /**
8300  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8301  * @hba: pointer to Host Bus Adapter (HBA)
8302  */
8303 void ufshcd_dealloc_host(struct ufs_hba *hba)
8304 {
8305         scsi_host_put(hba->host);
8306 }
8307 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8308
8309 /**
8310  * ufshcd_set_dma_mask - Set dma mask based on the controller
8311  *                       addressing capability
8312  * @hba: per adapter instance
8313  *
8314  * Returns 0 for success, non-zero for failure
8315  */
8316 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8317 {
8318         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8319                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8320                         return 0;
8321         }
8322         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8323 }
8324
8325 /**
8326  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
8327  * @dev: pointer to device handle
8328  * @hba_handle: driver private handle
8329  * Returns 0 on success, non-zero value on failure
8330  */
8331 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
8332 {
8333         struct Scsi_Host *host;
8334         struct ufs_hba *hba;
8335         int err = 0;
8336
8337         if (!dev) {
8338                 dev_err(dev,
8339                 "Invalid memory reference for dev is NULL\n");
8340                 err = -ENODEV;
8341                 goto out_error;
8342         }
8343
8344         host = scsi_host_alloc(&ufshcd_driver_template,
8345                                 sizeof(struct ufs_hba));
8346         if (!host) {
8347                 dev_err(dev, "scsi_host_alloc failed\n");
8348                 err = -ENOMEM;
8349                 goto out_error;
8350         }
8351         hba = shost_priv(host);
8352         hba->host = host;
8353         hba->dev = dev;
8354         *hba_handle = hba;
8355         hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
8356
8357         INIT_LIST_HEAD(&hba->clk_list_head);
8358
8359 out_error:
8360         return err;
8361 }
8362 EXPORT_SYMBOL(ufshcd_alloc_host);
8363
8364 /* This function exists because blk_mq_alloc_tag_set() requires this. */
8365 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
8366                                      const struct blk_mq_queue_data *qd)
8367 {
8368         WARN_ON_ONCE(true);
8369         return BLK_STS_NOTSUPP;
8370 }
8371
8372 static const struct blk_mq_ops ufshcd_tmf_ops = {
8373         .queue_rq = ufshcd_queue_tmf,
8374 };
8375
8376 /**
8377  * ufshcd_init - Driver initialization routine
8378  * @hba: per-adapter instance
8379  * @mmio_base: base register address
8380  * @irq: Interrupt line of device
8381  * Returns 0 on success, non-zero value on failure
8382  */
8383 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8384 {
8385         int err;
8386         struct Scsi_Host *host = hba->host;
8387         struct device *dev = hba->dev;
8388
8389         if (!mmio_base) {
8390                 dev_err(hba->dev,
8391                 "Invalid memory reference for mmio_base is NULL\n");
8392                 err = -ENODEV;
8393                 goto out_error;
8394         }
8395
8396         hba->mmio_base = mmio_base;
8397         hba->irq = irq;
8398
8399         err = ufshcd_hba_init(hba);
8400         if (err)
8401                 goto out_error;
8402
8403         /* Read capabilities registers */
8404         ufshcd_hba_capabilities(hba);
8405
8406         /* Get UFS version supported by the controller */
8407         hba->ufs_version = ufshcd_get_ufs_version(hba);
8408
8409         if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8410             (hba->ufs_version != UFSHCI_VERSION_11) &&
8411             (hba->ufs_version != UFSHCI_VERSION_20) &&
8412             (hba->ufs_version != UFSHCI_VERSION_21))
8413                 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8414                         hba->ufs_version);
8415
8416         /* Get Interrupt bit mask per version */
8417         hba->intr_mask = ufshcd_get_intr_mask(hba);
8418
8419         err = ufshcd_set_dma_mask(hba);
8420         if (err) {
8421                 dev_err(hba->dev, "set dma mask failed\n");
8422                 goto out_disable;
8423         }
8424
8425         /* Allocate memory for host memory space */
8426         err = ufshcd_memory_alloc(hba);
8427         if (err) {
8428                 dev_err(hba->dev, "Memory allocation failed\n");
8429                 goto out_disable;
8430         }
8431
8432         /* Configure LRB */
8433         ufshcd_host_memory_configure(hba);
8434
8435         host->can_queue = hba->nutrs;
8436         host->cmd_per_lun = hba->nutrs;
8437         host->max_id = UFSHCD_MAX_ID;
8438         host->max_lun = UFS_MAX_LUNS;
8439         host->max_channel = UFSHCD_MAX_CHANNEL;
8440         host->unique_id = host->host_no;
8441         host->max_cmd_len = UFS_CDB_SIZE;
8442
8443         hba->max_pwr_info.is_valid = false;
8444
8445         /* Initialize work queues */
8446         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
8447         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
8448
8449         /* Initialize UIC command mutex */
8450         mutex_init(&hba->uic_cmd_mutex);
8451
8452         /* Initialize mutex for device management commands */
8453         mutex_init(&hba->dev_cmd.lock);
8454
8455         init_rwsem(&hba->clk_scaling_lock);
8456
8457         ufshcd_init_clk_gating(hba);
8458
8459         ufshcd_init_clk_scaling(hba);
8460
8461         /*
8462          * In order to avoid any spurious interrupt immediately after
8463          * registering UFS controller interrupt handler, clear any pending UFS
8464          * interrupt status and disable all the UFS interrupts.
8465          */
8466         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8467                       REG_INTERRUPT_STATUS);
8468         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8469         /*
8470          * Make sure that UFS interrupts are disabled and any pending interrupt
8471          * status is cleared before registering UFS interrupt handler.
8472          */
8473         mb();
8474
8475         /* IRQ registration */
8476         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
8477         if (err) {
8478                 dev_err(hba->dev, "request irq failed\n");
8479                 goto exit_gating;
8480         } else {
8481                 hba->is_irq_enabled = true;
8482         }
8483
8484         err = scsi_add_host(host, hba->dev);
8485         if (err) {
8486                 dev_err(hba->dev, "scsi_add_host failed\n");
8487                 goto exit_gating;
8488         }
8489
8490         hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
8491         if (IS_ERR(hba->cmd_queue)) {
8492                 err = PTR_ERR(hba->cmd_queue);
8493                 goto out_remove_scsi_host;
8494         }
8495
8496         hba->tmf_tag_set = (struct blk_mq_tag_set) {
8497                 .nr_hw_queues   = 1,
8498                 .queue_depth    = hba->nutmrs,
8499                 .ops            = &ufshcd_tmf_ops,
8500                 .flags          = BLK_MQ_F_NO_SCHED,
8501         };
8502         err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
8503         if (err < 0)
8504                 goto free_cmd_queue;
8505         hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
8506         if (IS_ERR(hba->tmf_queue)) {
8507                 err = PTR_ERR(hba->tmf_queue);
8508                 goto free_tmf_tag_set;
8509         }
8510
8511         /* Reset the attached device */
8512         ufshcd_vops_device_reset(hba);
8513
8514         /* Host controller enable */
8515         err = ufshcd_hba_enable(hba);
8516         if (err) {
8517                 dev_err(hba->dev, "Host controller enable failed\n");
8518                 ufshcd_print_host_regs(hba);
8519                 ufshcd_print_host_state(hba);
8520                 goto free_tmf_queue;
8521         }
8522
8523         /*
8524          * Set the default power management level for runtime and system PM.
8525          * Default power saving mode is to keep UFS link in Hibern8 state
8526          * and UFS device in sleep state.
8527          */
8528         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8529                                                 UFS_SLEEP_PWR_MODE,
8530                                                 UIC_LINK_HIBERN8_STATE);
8531         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8532                                                 UFS_SLEEP_PWR_MODE,
8533                                                 UIC_LINK_HIBERN8_STATE);
8534
8535         /* Set the default auto-hiberate idle timer value to 150 ms */
8536         if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
8537                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8538                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8539         }
8540
8541         /* Hold auto suspend until async scan completes */
8542         pm_runtime_get_sync(dev);
8543         atomic_set(&hba->scsi_block_reqs_cnt, 0);
8544         /*
8545          * We are assuming that device wasn't put in sleep/power-down
8546          * state exclusively during the boot stage before kernel.
8547          * This assumption helps avoid doing link startup twice during
8548          * ufshcd_probe_hba().
8549          */
8550         ufshcd_set_ufs_dev_active(hba);
8551
8552         async_schedule(ufshcd_async_scan, hba);
8553         ufs_sysfs_add_nodes(hba->dev);
8554
8555         return 0;
8556
8557 free_tmf_queue:
8558         blk_cleanup_queue(hba->tmf_queue);
8559 free_tmf_tag_set:
8560         blk_mq_free_tag_set(&hba->tmf_tag_set);
8561 free_cmd_queue:
8562         blk_cleanup_queue(hba->cmd_queue);
8563 out_remove_scsi_host:
8564         scsi_remove_host(hba->host);
8565 exit_gating:
8566         ufshcd_exit_clk_scaling(hba);
8567         ufshcd_exit_clk_gating(hba);
8568 out_disable:
8569         hba->is_irq_enabled = false;
8570         ufshcd_hba_exit(hba);
8571 out_error:
8572         return err;
8573 }
8574 EXPORT_SYMBOL_GPL(ufshcd_init);
8575
8576 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8577 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
8578 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
8579 MODULE_LICENSE("GPL");
8580 MODULE_VERSION(UFSHCD_DRIVER_VERSION);