drivers/ufs/core/ufshcd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Universal Flash Storage Host controller driver Core
   4  * Copyright (C) 2011-2013 Samsung India Software Operations
   5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   6  *
   7  * Authors:
   8  *      Santosh Yaraganavi <santosh.sy@samsung.com>
   9  *      Vinayak Holikatti <h.vinayak@samsung.com>
  10  */
  11
  12 #include <linux/async.h>
  13 #include <linux/devfreq.h>
  14 #include <linux/nls.h>
  15 #include <linux/of.h>
  16 #include <linux/bitfield.h>
  17 #include <linux/blk-pm.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/clk.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/module.h>
  23 #include <linux/regulator/consumer.h>
  24 #include <scsi/scsi_cmnd.h>
  25 #include <scsi/scsi_dbg.h>
  26 #include <scsi/scsi_driver.h>
  27 #include <scsi/scsi_eh.h>
  28 #include "ufshcd-priv.h"
  29 #include <ufs/ufs_quirks.h>
  30 #include <ufs/unipro.h>
  31 #include "ufs-sysfs.h"
  32 #include "ufs-debugfs.h"
  33 #include "ufs-fault-injection.h"
  34 #include "ufs_bsg.h"
  35 #include "ufshcd-crypto.h"
  36 #include "ufshpb.h"
  37 #include <asm/unaligned.h>
  38
  39 #define CREATE_TRACE_POINTS
  40 #include <trace/events/ufs.h>
  41
  42 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  43                                  UTP_TASK_REQ_COMPL |\
  44                                  UFSHCD_ERROR_MASK)
  45 /* UIC command timeout, unit: ms */
  46 #define UIC_CMD_TIMEOUT 500
  47
  48 /* NOP OUT retries waiting for NOP IN response */
  49 #define NOP_OUT_RETRIES    10
  50 /* Timeout after 50 msecs if NOP OUT hangs without response */
  51 #define NOP_OUT_TIMEOUT    50 /* msecs */
  52
  53 /* Query request retries */
  54 #define QUERY_REQ_RETRIES 3
  55 /* Query request timeout */
  56 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  57
  58 /* Task management command timeout */
  59 #define TM_CMD_TIMEOUT  100 /* msecs */
  60
  61 /* maximum number of retries for a general UIC command  */
  62 #define UFS_UIC_COMMAND_RETRIES 3
  63
  64 /* maximum number of link-startup retries */
  65 #define DME_LINKSTARTUP_RETRIES 3
  66
  67 /* Maximum retries for Hibern8 enter */
  68 #define UIC_HIBERN8_ENTER_RETRIES 3
  69
  70 /* maximum number of reset retries before giving up */
  71 #define MAX_HOST_RESET_RETRIES 5
  72
  73 /* Maximum number of error handler retries before giving up */
  74 #define MAX_ERR_HANDLER_RETRIES 5
  75
  76 /* Expose the flag value from utp_upiu_query.value */
  77 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  78
  79 /* Interrupt aggregation default timeout, unit: 40us */
  80 #define INT_AGGR_DEF_TO 0x02
  81
  82 /* default delay of autosuspend: 2000 ms */
  83 #define RPM_AUTOSUSPEND_DELAY_MS 2000
  84
  85 /* Default delay of RPM device flush delayed work */
  86 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
  87
  88 /* Default value of wait time before gating device ref clock */
  89 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
  90
  91 /* Polling time to wait for fDeviceInit */
  92 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
  93
  94 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  95         ({                                                              \
  96                 int _ret;                                               \
  97                 if (_on)                                                \
  98                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  99                 else                                                    \
 100                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
 101                 _ret;                                                   \
 102         })
 103
 104 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
 105         size_t __len = (len);                                            \
 106         print_hex_dump(KERN_ERR, prefix_str,                             \
 107                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
 108                        16, 4, buf, __len, false);                        \
 109 } while (0)
 110
 111 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 112                      const char *prefix)
 113 {
 114         u32 *regs;
 115         size_t pos;
 116
 117         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 118                 return -EINVAL;
 119
 120         regs = kzalloc(len, GFP_ATOMIC);
 121         if (!regs)
 122                 return -ENOMEM;
 123
 124         for (pos = 0; pos < len; pos += 4) {
 125                 if (offset == 0 &&
 126                     pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
 127                     pos <= REG_UIC_ERROR_CODE_DME)
 128                         continue;
 129                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 130         }
 131
 132         ufshcd_hex_dump(prefix, regs, len);
 133         kfree(regs);
 134
 135         return 0;
 136 }
 137 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 138
 139 enum {
 140         UFSHCD_MAX_CHANNEL      = 0,
 141         UFSHCD_MAX_ID           = 1,
 142         UFSHCD_NUM_RESERVED     = 1,
 143         UFSHCD_CMD_PER_LUN      = 32 - UFSHCD_NUM_RESERVED,
 144         UFSHCD_CAN_QUEUE        = 32 - UFSHCD_NUM_RESERVED,
 145 };
 146
 147 static const char *const ufshcd_state_name[] = {
 148         [UFSHCD_STATE_RESET]                    = "reset",
 149         [UFSHCD_STATE_OPERATIONAL]              = "operational",
 150         [UFSHCD_STATE_ERROR]                    = "error",
 151         [UFSHCD_STATE_EH_SCHEDULED_FATAL]       = "eh_fatal",
 152         [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL]   = "eh_non_fatal",
 153 };
 154
 155 /* UFSHCD error handling flags */
 156 enum {
 157         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 158 };
 159
 160 /* UFSHCD UIC layer error flags */
 161 enum {
 162         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 163         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 164         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 165         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 166         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 167         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 168         UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
 169 };
 170
 171 #define ufshcd_set_eh_in_progress(h) \
 172         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 173 #define ufshcd_eh_in_progress(h) \
 174         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 175 #define ufshcd_clear_eh_in_progress(h) \
 176         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 177
 178 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 179         [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 180         [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 181         [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 182         [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 183         [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 184         [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 185         /*
 186          * For DeepSleep, the link is first put in hibern8 and then off.
 187          * Leaving the link in hibern8 is not supported.
 188          */
 189         [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
 190 };
 191
 192 static inline enum ufs_dev_pwr_mode
 193 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 194 {
 195         return ufs_pm_lvl_states[lvl].dev_state;
 196 }
 197
 198 static inline enum uic_link_state
 199 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 200 {
 201         return ufs_pm_lvl_states[lvl].link_state;
 202 }
 203
 204 static inline enum ufs_pm_level
 205 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 206                                         enum uic_link_state link_state)
 207 {
 208         enum ufs_pm_level lvl;
 209
 210         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 211                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 212                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 213                         return lvl;
 214         }
 215
 216         /* if no match found, return the level 0 */
 217         return UFS_PM_LVL_0;
 218 }
 219
 220 static const struct ufs_dev_quirk ufs_fixups[] = {
 221         /* UFS cards deviations table */
 222         { .wmanufacturerid = UFS_VENDOR_MICRON,
 223           .model = UFS_ANY_MODEL,
 224           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 225                    UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ },
 226         { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
 227           .model = UFS_ANY_MODEL,
 228           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 229                    UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
 230                    UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
 231         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 232           .model = UFS_ANY_MODEL,
 233           .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
 234         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 235           .model = "hB8aL1" /*H28U62301AMR*/,
 236           .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
 237         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 238           .model = UFS_ANY_MODEL,
 239           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
 240         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 241           .model = "THGLF2G9C8KBADG",
 242           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 243         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 244           .model = "THGLF2G9D8KBADG",
 245           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 246         {}
 247 };
 248
 249 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 250 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 251 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 252 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 253 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 254 static void ufshcd_hba_exit(struct ufs_hba *hba);
 255 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
 256 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 257 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 258 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 259 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 260 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 261 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 262 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 263 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 264 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 265                              struct ufs_pa_layer_attr *pwr_mode);
 266 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 267 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 268 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 269                                          struct ufs_vreg *vreg);
 270 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
 271 static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
 272 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
 273 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
 274 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
 275
 276 static inline void ufshcd_enable_irq(struct ufs_hba *hba)
 277 {
 278         if (!hba->is_irq_enabled) {
 279                 enable_irq(hba->irq);
 280                 hba->is_irq_enabled = true;
 281         }
 282 }
 283
 284 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 285 {
 286         if (hba->is_irq_enabled) {
 287                 disable_irq(hba->irq);
 288                 hba->is_irq_enabled = false;
 289         }
 290 }
 291
 292 static inline void ufshcd_wb_config(struct ufs_hba *hba)
 293 {
 294         if (!ufshcd_is_wb_allowed(hba))
 295                 return;
 296
 297         ufshcd_wb_toggle(hba, true);
 298
 299         ufshcd_wb_toggle_flush_during_h8(hba, true);
 300         if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
 301                 ufshcd_wb_toggle_flush(hba, true);
 302 }
 303
 304 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 305 {
 306         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 307                 scsi_unblock_requests(hba->host);
 308 }
 309
 310 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 311 {
 312         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 313                 scsi_block_requests(hba->host);
 314 }
 315
 316 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 317                                       enum ufs_trace_str_t str_t)
 318 {
 319         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 320         struct utp_upiu_header *header;
 321
 322         if (!trace_ufshcd_upiu_enabled())
 323                 return;
 324
 325         if (str_t == UFS_CMD_SEND)
 326                 header = &rq->header;
 327         else
 328                 header = &hba->lrb[tag].ucd_rsp_ptr->header;
 329
 330         trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
 331                           UFS_TSF_CDB);
 332 }
 333
 334 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
 335                                         enum ufs_trace_str_t str_t,
 336                                         struct utp_upiu_req *rq_rsp)
 337 {
 338         if (!trace_ufshcd_upiu_enabled())
 339                 return;
 340
 341         trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
 342                           &rq_rsp->qr, UFS_TSF_OSF);
 343 }
 344
 345 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 346                                      enum ufs_trace_str_t str_t)
 347 {
 348         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
 349
 350         if (!trace_ufshcd_upiu_enabled())
 351                 return;
 352
 353         if (str_t == UFS_TM_SEND)
 354                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 355                                   &descp->upiu_req.req_header,
 356                                   &descp->upiu_req.input_param1,
 357                                   UFS_TSF_TM_INPUT);
 358         else
 359                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 360                                   &descp->upiu_rsp.rsp_header,
 361                                   &descp->upiu_rsp.output_param1,
 362                                   UFS_TSF_TM_OUTPUT);
 363 }
 364
 365 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
 366                                          struct uic_command *ucmd,
 367                                          enum ufs_trace_str_t str_t)
 368 {
 369         u32 cmd;
 370
 371         if (!trace_ufshcd_uic_command_enabled())
 372                 return;
 373
 374         if (str_t == UFS_CMD_SEND)
 375                 cmd = ucmd->command;
 376         else
 377                 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
 378
 379         trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
 380                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
 381                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
 382                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
 383 }
 384
 385 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 386                                      enum ufs_trace_str_t str_t)
 387 {
 388         u64 lba = 0;
 389         u8 opcode = 0, group_id = 0;
 390         u32 intr, doorbell;
 391         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 392         struct scsi_cmnd *cmd = lrbp->cmd;
 393         struct request *rq = scsi_cmd_to_rq(cmd);
 394         int transfer_len = -1;
 395
 396         if (!cmd)
 397                 return;
 398
 399         /* trace UPIU also */
 400         ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 401         if (!trace_ufshcd_command_enabled())
 402                 return;
 403
 404         opcode = cmd->cmnd[0];
 405
 406         if (opcode == READ_10 || opcode == WRITE_10) {
 407                 /*
 408                  * Currently we only fully trace read(10) and write(10) commands
 409                  */
 410                 transfer_len =
 411                        be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 412                 lba = scsi_get_lba(cmd);
 413                 if (opcode == WRITE_10)
 414                         group_id = lrbp->cmd->cmnd[6];
 415         } else if (opcode == UNMAP) {
 416                 /*
 417                  * The number of Bytes to be unmapped beginning with the lba.
 418                  */
 419                 transfer_len = blk_rq_bytes(rq);
 420                 lba = scsi_get_lba(cmd);
 421         }
 422
 423         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 424         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 425         trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
 426                         doorbell, transfer_len, intr, lba, opcode, group_id);
 427 }
 428
 429 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 430 {
 431         struct ufs_clk_info *clki;
 432         struct list_head *head = &hba->clk_list_head;
 433
 434         if (list_empty(head))
 435                 return;
 436
 437         list_for_each_entry(clki, head, list) {
 438                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 439                                 clki->max_freq)
 440                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 441                                         clki->name, clki->curr_freq);
 442         }
 443 }
 444
 445 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
 446                              char *err_name)
 447 {
 448         int i;
 449         bool found = false;
 450         struct ufs_event_hist *e;
 451
 452         if (id >= UFS_EVT_CNT)
 453                 return;
 454
 455         e = &hba->ufs_stats.event[id];
 456
 457         for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
 458                 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
 459
 460                 if (e->tstamp[p] == 0)
 461                         continue;
 462                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 463                         e->val[p], ktime_to_us(e->tstamp[p]));
 464                 found = true;
 465         }
 466
 467         if (!found)
 468                 dev_err(hba->dev, "No record of %s\n", err_name);
 469         else
 470                 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
 471 }
 472
 473 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 474 {
 475         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 476
 477         ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
 478         ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
 479         ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
 480         ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
 481         ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
 482         ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
 483                          "auto_hibern8_err");
 484         ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
 485         ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
 486                          "link_startup_fail");
 487         ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
 488         ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
 489                          "suspend_fail");
 490         ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
 491         ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
 492         ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
 493
 494         ufshcd_vops_dbg_register_dump(hba);
 495 }
 496
 497 static
 498 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
 499 {
 500         struct ufshcd_lrb *lrbp;
 501         int prdt_length;
 502         int tag;
 503
 504         for_each_set_bit(tag, &bitmap, hba->nutrs) {
 505                 lrbp = &hba->lrb[tag];
 506
 507                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 508                                 tag, ktime_to_us(lrbp->issue_time_stamp));
 509                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 510                                 tag, ktime_to_us(lrbp->compl_time_stamp));
 511                 dev_err(hba->dev,
 512                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 513                         tag, (u64)lrbp->utrd_dma_addr);
 514
 515                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 516                                 sizeof(struct utp_transfer_req_desc));
 517                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 518                         (u64)lrbp->ucd_req_dma_addr);
 519                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 520                                 sizeof(struct utp_upiu_req));
 521                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 522                         (u64)lrbp->ucd_rsp_dma_addr);
 523                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 524                                 sizeof(struct utp_upiu_rsp));
 525
 526                 prdt_length = le16_to_cpu(
 527                         lrbp->utr_descriptor_ptr->prd_table_length);
 528                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
 529                         prdt_length /= sizeof(struct ufshcd_sg_entry);
 530
 531                 dev_err(hba->dev,
 532                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 533                         tag, prdt_length,
 534                         (u64)lrbp->ucd_prdt_dma_addr);
 535
 536                 if (pr_prdt)
 537                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 538                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
 539         }
 540 }
 541
 542 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 543 {
 544         int tag;
 545
 546         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 547                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 548
 549                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 550                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 551         }
 552 }
 553
 554 static void ufshcd_print_host_state(struct ufs_hba *hba)
 555 {
 556         struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
 557
 558         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 559         dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 560                 hba->outstanding_reqs, hba->outstanding_tasks);
 561         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 562                 hba->saved_err, hba->saved_uic_err);
 563         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 564                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 565         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 566                 hba->pm_op_in_progress, hba->is_sys_suspended);
 567         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 568                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 569         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 570         dev_err(hba->dev,
 571                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
 572                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
 573                 hba->ufs_stats.hibern8_exit_cnt);
 574         dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
 575                 ktime_to_us(hba->ufs_stats.last_intr_ts),
 576                 hba->ufs_stats.last_intr_status);
 577         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 578                 hba->eh_flags, hba->req_abort_count);
 579         dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
 580                 hba->ufs_version, hba->capabilities, hba->caps);
 581         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 582                 hba->dev_quirks);
 583         if (sdev_ufs)
 584                 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
 585                         sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
 586
 587         ufshcd_print_clk_freqs(hba);
 588 }
 589
 590 /**
 591  * ufshcd_print_pwr_info - print power params as saved in hba
 592  * power info
 593  * @hba: per-adapter instance
 594  */
 595 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 596 {
 597         static const char * const names[] = {
 598                 "INVALID MODE",
 599                 "FAST MODE",
 600                 "SLOW_MODE",
 601                 "INVALID MODE",
 602                 "FASTAUTO_MODE",
 603                 "SLOWAUTO_MODE",
 604                 "INVALID MODE",
 605         };
 606
 607         /*
 608          * Using dev_dbg to avoid messages during runtime PM to avoid
 609          * never-ending cycles of messages written back to storage by user space
 610          * causing runtime resume, causing more messages and so on.
 611          */
 612         dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 613                  __func__,
 614                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 615                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 616                  names[hba->pwr_info.pwr_rx],
 617                  names[hba->pwr_info.pwr_tx],
 618                  hba->pwr_info.hs_rate);
 619 }
 620
 621 static void ufshcd_device_reset(struct ufs_hba *hba)
 622 {
 623         int err;
 624
 625         err = ufshcd_vops_device_reset(hba);
 626
 627         if (!err) {
 628                 ufshcd_set_ufs_dev_active(hba);
 629                 if (ufshcd_is_wb_allowed(hba)) {
 630                         hba->dev_info.wb_enabled = false;
 631                         hba->dev_info.wb_buf_flush_enabled = false;
 632                 }
 633         }
 634         if (err != -EOPNOTSUPP)
 635                 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
 636 }
 637
 638 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 639 {
 640         if (!us)
 641                 return;
 642
 643         if (us < 10)
 644                 udelay(us);
 645         else
 646                 usleep_range(us, us + tolerance);
 647 }
 648 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 649
 650 /**
 651  * ufshcd_wait_for_register - wait for register value to change
 652  * @hba: per-adapter interface
 653  * @reg: mmio register offset
 654  * @mask: mask to apply to the read register value
 655  * @val: value to wait for
 656  * @interval_us: polling interval in microseconds
 657  * @timeout_ms: timeout in milliseconds
 658  *
 659  * Return:
 660  * -ETIMEDOUT on error, zero on success.
 661  */
 662 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 663                                 u32 val, unsigned long interval_us,
 664                                 unsigned long timeout_ms)
 665 {
 666         int err = 0;
 667         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 668
 669         /* ignore bits that we don't intend to wait on */
 670         val = val & mask;
 671
 672         while ((ufshcd_readl(hba, reg) & mask) != val) {
 673                 usleep_range(interval_us, interval_us + 50);
 674                 if (time_after(jiffies, timeout)) {
 675                         if ((ufshcd_readl(hba, reg) & mask) != val)
 676                                 err = -ETIMEDOUT;
 677                         break;
 678                 }
 679         }
 680
 681         return err;
 682 }
 683
 684 /**
 685  * ufshcd_get_intr_mask - Get the interrupt bit mask
 686  * @hba: Pointer to adapter instance
 687  *
 688  * Returns interrupt bit mask per version
 689  */
 690 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 691 {
 692         if (hba->ufs_version == ufshci_version(1, 0))
 693                 return INTERRUPT_MASK_ALL_VER_10;
 694         if (hba->ufs_version <= ufshci_version(2, 0))
 695                 return INTERRUPT_MASK_ALL_VER_11;
 696
 697         return INTERRUPT_MASK_ALL_VER_21;
 698 }
 699
 700 /**
 701  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 702  * @hba: Pointer to adapter instance
 703  *
 704  * Returns UFSHCI version supported by the controller
 705  */
 706 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 707 {
 708         u32 ufshci_ver;
 709
 710         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 711                 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
 712         else
 713                 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
 714
 715         /*
 716          * UFSHCI v1.x uses a different version scheme, in order
 717          * to allow the use of comparisons with the ufshci_version
 718          * function, we convert it to the same scheme as ufs 2.0+.
 719          */
 720         if (ufshci_ver & 0x00010000)
 721                 return ufshci_version(1, ufshci_ver & 0x00000100);
 722
 723         return ufshci_ver;
 724 }
 725
 726 /**
 727  * ufshcd_is_device_present - Check if any device connected to
 728  *                            the host controller
 729  * @hba: pointer to adapter instance
 730  *
 731  * Returns true if device present, false if no device detected
 732  */
 733 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 734 {
 735         return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
 736 }
 737
 738 /**
 739  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 740  * @lrbp: pointer to local command reference block
 741  *
 742  * This function is used to get the OCS field from UTRD
 743  * Returns the OCS field in the UTRD
 744  */
 745 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 746 {
 747         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 748 }
 749
 750 /**
 751  * ufshcd_utrl_clear() - Clear requests from the controller request list.
 752  * @hba: per adapter instance
 753  * @mask: mask with one bit set for each request to be cleared
 754  */
 755 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
 756 {
 757         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 758                 mask = ~mask;
 759         /*
 760          * From the UFSHCI specification: "UTP Transfer Request List CLear
 761          * Register (UTRLCLR): This field is bit significant. Each bit
 762          * corresponds to a slot in the UTP Transfer Request List, where bit 0
 763          * corresponds to request slot 0. A bit in this field is set to ‘0’
 764          * by host software to indicate to the host controller that a transfer
 765          * request slot is cleared. The host controller
 766          * shall free up any resources associated to the request slot
 767          * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
 768          * host software indicates no change to request slots by setting the
 769          * associated bits in this field to ‘1’. Bits in this field shall only
 770          * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
 771          */
 772         ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 773 }
 774
 775 /**
 776  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
 777  * @hba: per adapter instance
 778  * @pos: position of the bit to be cleared
 779  */
 780 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 781 {
 782         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 783                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 784         else
 785                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 786 }
 787
 788 /**
 789  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 790  * @reg: Register value of host controller status
 791  *
 792  * Returns integer, 0 on Success and positive value if failed
 793  */
 794 static inline int ufshcd_get_lists_status(u32 reg)
 795 {
 796         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 797 }
 798
 799 /**
 800  * ufshcd_get_uic_cmd_result - Get the UIC command result
 801  * @hba: Pointer to adapter instance
 802  *
 803  * This function gets the result of UIC command completion
 804  * Returns 0 on success, non zero value on error
 805  */
 806 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 807 {
 808         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 809                MASK_UIC_COMMAND_RESULT;
 810 }
 811
 812 /**
 813  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 814  * @hba: Pointer to adapter instance
 815  *
 816  * This function gets UIC command argument3
 817  * Returns 0 on success, non zero value on error
 818  */
 819 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 820 {
 821         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 822 }
 823
 824 /**
 825  * ufshcd_get_req_rsp - returns the TR response transaction type
 826  * @ucd_rsp_ptr: pointer to response UPIU
 827  */
 828 static inline int
 829 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 830 {
 831         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 832 }
 833
 834 /**
 835  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 836  * @ucd_rsp_ptr: pointer to response UPIU
 837  *
 838  * This function gets the response status and scsi_status from response UPIU
 839  * Returns the response result code.
 840  */
 841 static inline int
 842 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 843 {
 844         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 845 }
 846
 847 /*
 848  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 849  *                              from response UPIU
 850  * @ucd_rsp_ptr: pointer to response UPIU
 851  *
 852  * Return the data segment length.
 853  */
 854 static inline unsigned int
 855 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 856 {
 857         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 858                 MASK_RSP_UPIU_DATA_SEG_LEN;
 859 }
 860
 861 /**
 862  * ufshcd_is_exception_event - Check if the device raised an exception event
 863  * @ucd_rsp_ptr: pointer to response UPIU
 864  *
 865  * The function checks if the device raised an exception event indicated in
 866  * the Device Information field of response UPIU.
 867  *
 868  * Returns true if exception is raised, false otherwise.
 869  */
 870 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 871 {
 872         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 873                         MASK_RSP_EXCEPTION_EVENT;
 874 }
 875
 876 /**
 877  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 878  * @hba: per adapter instance
 879  */
 880 static inline void
 881 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 882 {
 883         ufshcd_writel(hba, INT_AGGR_ENABLE |
 884                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 885                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 886 }
 887
 888 /**
 889  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 890  * @hba: per adapter instance
 891  * @cnt: Interrupt aggregation counter threshold
 892  * @tmout: Interrupt aggregation timeout value
 893  */
 894 static inline void
 895 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 896 {
 897         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 898                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 899                       INT_AGGR_TIMEOUT_VAL(tmout),
 900                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 901 }
 902
 903 /**
 904  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 905  * @hba: per adapter instance
 906  */
 907 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 908 {
 909         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 910 }
 911
 912 /**
 913  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 914  *                      When run-stop registers are set to 1, it indicates the
 915  *                      host controller that it can process the requests
 916  * @hba: per adapter instance
 917  */
 918 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 919 {
 920         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 921                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 922         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 923                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 924 }
 925
 926 /**
 927  * ufshcd_hba_start - Start controller initialization sequence
 928  * @hba: per adapter instance
 929  */
 930 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 931 {
 932         u32 val = CONTROLLER_ENABLE;
 933
 934         if (ufshcd_crypto_enable(hba))
 935                 val |= CRYPTO_GENERAL_ENABLE;
 936
 937         ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
 938 }
 939
 940 /**
 941  * ufshcd_is_hba_active - Get controller state
 942  * @hba: per adapter instance
 943  *
 944  * Returns true if and only if the controller is active.
 945  */
 946 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 947 {
 948         return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
 949 }
 950
 951 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 952 {
 953         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 954         if (hba->ufs_version <= ufshci_version(1, 1))
 955                 return UFS_UNIPRO_VER_1_41;
 956         else
 957                 return UFS_UNIPRO_VER_1_6;
 958 }
 959 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
 960
 961 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 962 {
 963         /*
 964          * If both host and device support UniPro ver1.6 or later, PA layer
 965          * parameters tuning happens during link startup itself.
 966          *
 967          * We can manually tune PA layer parameters if either host or device
 968          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
 969          * logic simple, we will only do manual tuning if local unipro version
 970          * doesn't support ver1.6 or later.
 971          */
 972         return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6;
 973 }
 974
 975 /**
 976  * ufshcd_set_clk_freq - set UFS controller clock frequencies
 977  * @hba: per adapter instance
 978  * @scale_up: If True, set max possible frequency othewise set low frequency
 979  *
 980  * Returns 0 if successful
 981  * Returns < 0 for any other errors
 982  */
 983 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
 984 {
 985         int ret = 0;
 986         struct ufs_clk_info *clki;
 987         struct list_head *head = &hba->clk_list_head;
 988
 989         if (list_empty(head))
 990                 goto out;
 991
 992         list_for_each_entry(clki, head, list) {
 993                 if (!IS_ERR_OR_NULL(clki->clk)) {
 994                         if (scale_up && clki->max_freq) {
 995                                 if (clki->curr_freq == clki->max_freq)
 996                                         continue;
 997
 998                                 ret = clk_set_rate(clki->clk, clki->max_freq);
 999                                 if (ret) {
1000                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1001                                                 __func__, clki->name,
1002                                                 clki->max_freq, ret);
1003                                         break;
1004                                 }
1005                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1006                                                 "scaled up", clki->name,
1007                                                 clki->curr_freq,
1008                                                 clki->max_freq);
1009
1010                                 clki->curr_freq = clki->max_freq;
1011
1012                         } else if (!scale_up && clki->min_freq) {
1013                                 if (clki->curr_freq == clki->min_freq)
1014                                         continue;
1015
1016                                 ret = clk_set_rate(clki->clk, clki->min_freq);
1017                                 if (ret) {
1018                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1019                                                 __func__, clki->name,
1020                                                 clki->min_freq, ret);
1021                                         break;
1022                                 }
1023                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1024                                                 "scaled down", clki->name,
1025                                                 clki->curr_freq,
1026                                                 clki->min_freq);
1027                                 clki->curr_freq = clki->min_freq;
1028                         }
1029                 }
1030                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1031                                 clki->name, clk_get_rate(clki->clk));
1032         }
1033
1034 out:
1035         return ret;
1036 }
1037
1038 /**
1039  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1040  * @hba: per adapter instance
1041  * @scale_up: True if scaling up and false if scaling down
1042  *
1043  * Returns 0 if successful
1044  * Returns < 0 for any other errors
1045  */
1046 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1047 {
1048         int ret = 0;
1049         ktime_t start = ktime_get();
1050
1051         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1052         if (ret)
1053                 goto out;
1054
1055         ret = ufshcd_set_clk_freq(hba, scale_up);
1056         if (ret)
1057                 goto out;
1058
1059         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1060         if (ret)
1061                 ufshcd_set_clk_freq(hba, !scale_up);
1062
1063 out:
1064         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1065                         (scale_up ? "up" : "down"),
1066                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1067         return ret;
1068 }
1069
1070 /**
1071  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1072  * @hba: per adapter instance
1073  * @scale_up: True if scaling up and false if scaling down
1074  *
1075  * Returns true if scaling is required, false otherwise.
1076  */
1077 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1078                                                bool scale_up)
1079 {
1080         struct ufs_clk_info *clki;
1081         struct list_head *head = &hba->clk_list_head;
1082
1083         if (list_empty(head))
1084                 return false;
1085
1086         list_for_each_entry(clki, head, list) {
1087                 if (!IS_ERR_OR_NULL(clki->clk)) {
1088                         if (scale_up && clki->max_freq) {
1089                                 if (clki->curr_freq == clki->max_freq)
1090                                         continue;
1091                                 return true;
1092                         } else if (!scale_up && clki->min_freq) {
1093                                 if (clki->curr_freq == clki->min_freq)
1094                                         continue;
1095                                 return true;
1096                         }
1097                 }
1098         }
1099
1100         return false;
1101 }
1102
1103 /*
1104  * Determine the number of pending commands by counting the bits in the SCSI
1105  * device budget maps. This approach has been selected because a bit is set in
1106  * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1107  * flag. The host_self_blocked flag can be modified by calling
1108  * scsi_block_requests() or scsi_unblock_requests().
1109  */
1110 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1111 {
1112         struct scsi_device *sdev;
1113         u32 pending = 0;
1114
1115         lockdep_assert_held(hba->host->host_lock);
1116         __shost_for_each_device(sdev, hba->host)
1117                 pending += sbitmap_weight(&sdev->budget_map);
1118
1119         return pending;
1120 }
1121
1122 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1123                                         u64 wait_timeout_us)
1124 {
1125         unsigned long flags;
1126         int ret = 0;
1127         u32 tm_doorbell;
1128         u32 tr_pending;
1129         bool timeout = false, do_last_check = false;
1130         ktime_t start;
1131
1132         ufshcd_hold(hba, false);
1133         spin_lock_irqsave(hba->host->host_lock, flags);
1134         /*
1135          * Wait for all the outstanding tasks/transfer requests.
1136          * Verify by checking the doorbell registers are clear.
1137          */
1138         start = ktime_get();
1139         do {
1140                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1141                         ret = -EBUSY;
1142                         goto out;
1143                 }
1144
1145                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1146                 tr_pending = ufshcd_pending_cmds(hba);
1147                 if (!tm_doorbell && !tr_pending) {
1148                         timeout = false;
1149                         break;
1150                 } else if (do_last_check) {
1151                         break;
1152                 }
1153
1154                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1155                 schedule();
1156                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1157                     wait_timeout_us) {
1158                         timeout = true;
1159                         /*
1160                          * We might have scheduled out for long time so make
1161                          * sure to check if doorbells are cleared by this time
1162                          * or not.
1163                          */
1164                         do_last_check = true;
1165                 }
1166                 spin_lock_irqsave(hba->host->host_lock, flags);
1167         } while (tm_doorbell || tr_pending);
1168
1169         if (timeout) {
1170                 dev_err(hba->dev,
1171                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1172                         __func__, tm_doorbell, tr_pending);
1173                 ret = -EBUSY;
1174         }
1175 out:
1176         spin_unlock_irqrestore(hba->host->host_lock, flags);
1177         ufshcd_release(hba);
1178         return ret;
1179 }
1180
1181 /**
1182  * ufshcd_scale_gear - scale up/down UFS gear
1183  * @hba: per adapter instance
1184  * @scale_up: True for scaling up gear and false for scaling down
1185  *
1186  * Returns 0 for success,
1187  * Returns -EBUSY if scaling can't happen at this time
1188  * Returns non-zero for any other errors
1189  */
1190 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1191 {
1192         int ret = 0;
1193         struct ufs_pa_layer_attr new_pwr_info;
1194
1195         if (scale_up) {
1196                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1197                        sizeof(struct ufs_pa_layer_attr));
1198         } else {
1199                 memcpy(&new_pwr_info, &hba->pwr_info,
1200                        sizeof(struct ufs_pa_layer_attr));
1201
1202                 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1203                     hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1204                         /* save the current power mode */
1205                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1206                                 &hba->pwr_info,
1207                                 sizeof(struct ufs_pa_layer_attr));
1208
1209                         /* scale down gear */
1210                         new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1211                         new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1212                 }
1213         }
1214
1215         /* check if the power mode needs to be changed or not? */
1216         ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1217         if (ret)
1218                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1219                         __func__, ret,
1220                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1221                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1222
1223         return ret;
1224 }
1225
1226 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1227 {
1228         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1229         int ret = 0;
1230         /*
1231          * make sure that there are no outstanding requests when
1232          * clock scaling is in progress
1233          */
1234         ufshcd_scsi_block_requests(hba);
1235         down_write(&hba->clk_scaling_lock);
1236
1237         if (!hba->clk_scaling.is_allowed ||
1238             ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1239                 ret = -EBUSY;
1240                 up_write(&hba->clk_scaling_lock);
1241                 ufshcd_scsi_unblock_requests(hba);
1242                 goto out;
1243         }
1244
1245         /* let's not get into low power until clock scaling is completed */
1246         ufshcd_hold(hba, false);
1247
1248 out:
1249         return ret;
1250 }
1251
1252 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
1253 {
1254         if (writelock)
1255                 up_write(&hba->clk_scaling_lock);
1256         else
1257                 up_read(&hba->clk_scaling_lock);
1258         ufshcd_scsi_unblock_requests(hba);
1259         ufshcd_release(hba);
1260 }
1261
1262 /**
1263  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1264  * @hba: per adapter instance
1265  * @scale_up: True for scaling up and false for scalin down
1266  *
1267  * Returns 0 for success,
1268  * Returns -EBUSY if scaling can't happen at this time
1269  * Returns non-zero for any other errors
1270  */
1271 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1272 {
1273         int ret = 0;
1274         bool is_writelock = true;
1275
1276         ret = ufshcd_clock_scaling_prepare(hba);
1277         if (ret)
1278                 return ret;
1279
1280         /* scale down the gear before scaling down clocks */
1281         if (!scale_up) {
1282                 ret = ufshcd_scale_gear(hba, false);
1283                 if (ret)
1284                         goto out_unprepare;
1285         }
1286
1287         ret = ufshcd_scale_clks(hba, scale_up);
1288         if (ret) {
1289                 if (!scale_up)
1290                         ufshcd_scale_gear(hba, true);
1291                 goto out_unprepare;
1292         }
1293
1294         /* scale up the gear after scaling up clocks */
1295         if (scale_up) {
1296                 ret = ufshcd_scale_gear(hba, true);
1297                 if (ret) {
1298                         ufshcd_scale_clks(hba, false);
1299                         goto out_unprepare;
1300                 }
1301         }
1302
1303         /* Enable Write Booster if we have scaled up else disable it */
1304         downgrade_write(&hba->clk_scaling_lock);
1305         is_writelock = false;
1306         ufshcd_wb_toggle(hba, scale_up);
1307
1308 out_unprepare:
1309         ufshcd_clock_scaling_unprepare(hba, is_writelock);
1310         return ret;
1311 }
1312
1313 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1314 {
1315         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1316                                            clk_scaling.suspend_work);
1317         unsigned long irq_flags;
1318
1319         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1320         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1321                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1322                 return;
1323         }
1324         hba->clk_scaling.is_suspended = true;
1325         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1326
1327         __ufshcd_suspend_clkscaling(hba);
1328 }
1329
1330 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1331 {
1332         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1333                                            clk_scaling.resume_work);
1334         unsigned long irq_flags;
1335
1336         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1337         if (!hba->clk_scaling.is_suspended) {
1338                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1339                 return;
1340         }
1341         hba->clk_scaling.is_suspended = false;
1342         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1343
1344         devfreq_resume_device(hba->devfreq);
1345 }
1346
1347 static int ufshcd_devfreq_target(struct device *dev,
1348                                 unsigned long *freq, u32 flags)
1349 {
1350         int ret = 0;
1351         struct ufs_hba *hba = dev_get_drvdata(dev);
1352         ktime_t start;
1353         bool scale_up, sched_clk_scaling_suspend_work = false;
1354         struct list_head *clk_list = &hba->clk_list_head;
1355         struct ufs_clk_info *clki;
1356         unsigned long irq_flags;
1357
1358         if (!ufshcd_is_clkscaling_supported(hba))
1359                 return -EINVAL;
1360
1361         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1362         /* Override with the closest supported frequency */
1363         *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1364         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1365         if (ufshcd_eh_in_progress(hba)) {
1366                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1367                 return 0;
1368         }
1369
1370         if (!hba->clk_scaling.active_reqs)
1371                 sched_clk_scaling_suspend_work = true;
1372
1373         if (list_empty(clk_list)) {
1374                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1375                 goto out;
1376         }
1377
1378         /* Decide based on the rounded-off frequency and update */
1379         scale_up = *freq == clki->max_freq;
1380         if (!scale_up)
1381                 *freq = clki->min_freq;
1382         /* Update the frequency */
1383         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1384                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1385                 ret = 0;
1386                 goto out; /* no state change required */
1387         }
1388         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1389
1390         start = ktime_get();
1391         ret = ufshcd_devfreq_scale(hba, scale_up);
1392
1393         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1394                 (scale_up ? "up" : "down"),
1395                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1396
1397 out:
1398         if (sched_clk_scaling_suspend_work)
1399                 queue_work(hba->clk_scaling.workq,
1400                            &hba->clk_scaling.suspend_work);
1401
1402         return ret;
1403 }
1404
1405 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1406                 struct devfreq_dev_status *stat)
1407 {
1408         struct ufs_hba *hba = dev_get_drvdata(dev);
1409         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1410         unsigned long flags;
1411         struct list_head *clk_list = &hba->clk_list_head;
1412         struct ufs_clk_info *clki;
1413         ktime_t curr_t;
1414
1415         if (!ufshcd_is_clkscaling_supported(hba))
1416                 return -EINVAL;
1417
1418         memset(stat, 0, sizeof(*stat));
1419
1420         spin_lock_irqsave(hba->host->host_lock, flags);
1421         curr_t = ktime_get();
1422         if (!scaling->window_start_t)
1423                 goto start_window;
1424
1425         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1426         /*
1427          * If current frequency is 0, then the ondemand governor considers
1428          * there's no initial frequency set. And it always requests to set
1429          * to max. frequency.
1430          */
1431         stat->current_frequency = clki->curr_freq;
1432         if (scaling->is_busy_started)
1433                 scaling->tot_busy_t += ktime_us_delta(curr_t,
1434                                 scaling->busy_start_t);
1435
1436         stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1437         stat->busy_time = scaling->tot_busy_t;
1438 start_window:
1439         scaling->window_start_t = curr_t;
1440         scaling->tot_busy_t = 0;
1441
1442         if (hba->outstanding_reqs) {
1443                 scaling->busy_start_t = curr_t;
1444                 scaling->is_busy_started = true;
1445         } else {
1446                 scaling->busy_start_t = 0;
1447                 scaling->is_busy_started = false;
1448         }
1449         spin_unlock_irqrestore(hba->host->host_lock, flags);
1450         return 0;
1451 }
1452
1453 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1454 {
1455         struct list_head *clk_list = &hba->clk_list_head;
1456         struct ufs_clk_info *clki;
1457         struct devfreq *devfreq;
1458         int ret;
1459
1460         /* Skip devfreq if we don't have any clocks in the list */
1461         if (list_empty(clk_list))
1462                 return 0;
1463
1464         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1465         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1466         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1467
1468         ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1469                                          &hba->vps->ondemand_data);
1470         devfreq = devfreq_add_device(hba->dev,
1471                         &hba->vps->devfreq_profile,
1472                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1473                         &hba->vps->ondemand_data);
1474         if (IS_ERR(devfreq)) {
1475                 ret = PTR_ERR(devfreq);
1476                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1477
1478                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1479                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1480                 return ret;
1481         }
1482
1483         hba->devfreq = devfreq;
1484
1485         return 0;
1486 }
1487
1488 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1489 {
1490         struct list_head *clk_list = &hba->clk_list_head;
1491         struct ufs_clk_info *clki;
1492
1493         if (!hba->devfreq)
1494                 return;
1495
1496         devfreq_remove_device(hba->devfreq);
1497         hba->devfreq = NULL;
1498
1499         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1500         dev_pm_opp_remove(hba->dev, clki->min_freq);
1501         dev_pm_opp_remove(hba->dev, clki->max_freq);
1502 }
1503
1504 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1505 {
1506         unsigned long flags;
1507
1508         devfreq_suspend_device(hba->devfreq);
1509         spin_lock_irqsave(hba->host->host_lock, flags);
1510         hba->clk_scaling.window_start_t = 0;
1511         spin_unlock_irqrestore(hba->host->host_lock, flags);
1512 }
1513
1514 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1515 {
1516         unsigned long flags;
1517         bool suspend = false;
1518
1519         cancel_work_sync(&hba->clk_scaling.suspend_work);
1520         cancel_work_sync(&hba->clk_scaling.resume_work);
1521
1522         spin_lock_irqsave(hba->host->host_lock, flags);
1523         if (!hba->clk_scaling.is_suspended) {
1524                 suspend = true;
1525                 hba->clk_scaling.is_suspended = true;
1526         }
1527         spin_unlock_irqrestore(hba->host->host_lock, flags);
1528
1529         if (suspend)
1530                 __ufshcd_suspend_clkscaling(hba);
1531 }
1532
1533 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1534 {
1535         unsigned long flags;
1536         bool resume = false;
1537
1538         spin_lock_irqsave(hba->host->host_lock, flags);
1539         if (hba->clk_scaling.is_suspended) {
1540                 resume = true;
1541                 hba->clk_scaling.is_suspended = false;
1542         }
1543         spin_unlock_irqrestore(hba->host->host_lock, flags);
1544
1545         if (resume)
1546                 devfreq_resume_device(hba->devfreq);
1547 }
1548
1549 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1550                 struct device_attribute *attr, char *buf)
1551 {
1552         struct ufs_hba *hba = dev_get_drvdata(dev);
1553
1554         return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1555 }
1556
1557 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1558                 struct device_attribute *attr, const char *buf, size_t count)
1559 {
1560         struct ufs_hba *hba = dev_get_drvdata(dev);
1561         u32 value;
1562         int err = 0;
1563
1564         if (kstrtou32(buf, 0, &value))
1565                 return -EINVAL;
1566
1567         down(&hba->host_sem);
1568         if (!ufshcd_is_user_access_allowed(hba)) {
1569                 err = -EBUSY;
1570                 goto out;
1571         }
1572
1573         value = !!value;
1574         if (value == hba->clk_scaling.is_enabled)
1575                 goto out;
1576
1577         ufshcd_rpm_get_sync(hba);
1578         ufshcd_hold(hba, false);
1579
1580         hba->clk_scaling.is_enabled = value;
1581
1582         if (value) {
1583                 ufshcd_resume_clkscaling(hba);
1584         } else {
1585                 ufshcd_suspend_clkscaling(hba);
1586                 err = ufshcd_devfreq_scale(hba, true);
1587                 if (err)
1588                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1589                                         __func__, err);
1590         }
1591
1592         ufshcd_release(hba);
1593         ufshcd_rpm_put_sync(hba);
1594 out:
1595         up(&hba->host_sem);
1596         return err ? err : count;
1597 }
1598
1599 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1600 {
1601         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1602         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1603         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1604         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1605         hba->clk_scaling.enable_attr.attr.mode = 0644;
1606         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1607                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1608 }
1609
1610 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1611 {
1612         if (hba->clk_scaling.enable_attr.attr.name)
1613                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1614 }
1615
1616 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1617 {
1618         char wq_name[sizeof("ufs_clkscaling_00")];
1619
1620         if (!ufshcd_is_clkscaling_supported(hba))
1621                 return;
1622
1623         if (!hba->clk_scaling.min_gear)
1624                 hba->clk_scaling.min_gear = UFS_HS_G1;
1625
1626         INIT_WORK(&hba->clk_scaling.suspend_work,
1627                   ufshcd_clk_scaling_suspend_work);
1628         INIT_WORK(&hba->clk_scaling.resume_work,
1629                   ufshcd_clk_scaling_resume_work);
1630
1631         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1632                  hba->host->host_no);
1633         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1634
1635         hba->clk_scaling.is_initialized = true;
1636 }
1637
1638 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1639 {
1640         if (!hba->clk_scaling.is_initialized)
1641                 return;
1642
1643         ufshcd_remove_clk_scaling_sysfs(hba);
1644         destroy_workqueue(hba->clk_scaling.workq);
1645         ufshcd_devfreq_remove(hba);
1646         hba->clk_scaling.is_initialized = false;
1647 }
1648
1649 static void ufshcd_ungate_work(struct work_struct *work)
1650 {
1651         int ret;
1652         unsigned long flags;
1653         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1654                         clk_gating.ungate_work);
1655
1656         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1657
1658         spin_lock_irqsave(hba->host->host_lock, flags);
1659         if (hba->clk_gating.state == CLKS_ON) {
1660                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1661                 goto unblock_reqs;
1662         }
1663
1664         spin_unlock_irqrestore(hba->host->host_lock, flags);
1665         ufshcd_hba_vreg_set_hpm(hba);
1666         ufshcd_setup_clocks(hba, true);
1667
1668         ufshcd_enable_irq(hba);
1669
1670         /* Exit from hibern8 */
1671         if (ufshcd_can_hibern8_during_gating(hba)) {
1672                 /* Prevent gating in this path */
1673                 hba->clk_gating.is_suspended = true;
1674                 if (ufshcd_is_link_hibern8(hba)) {
1675                         ret = ufshcd_uic_hibern8_exit(hba);
1676                         if (ret)
1677                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1678                                         __func__, ret);
1679                         else
1680                                 ufshcd_set_link_active(hba);
1681                 }
1682                 hba->clk_gating.is_suspended = false;
1683         }
1684 unblock_reqs:
1685         ufshcd_scsi_unblock_requests(hba);
1686 }
1687
1688 /**
1689  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1690  * Also, exit from hibern8 mode and set the link as active.
1691  * @hba: per adapter instance
1692  * @async: This indicates whether caller should ungate clocks asynchronously.
1693  */
1694 int ufshcd_hold(struct ufs_hba *hba, bool async)
1695 {
1696         int rc = 0;
1697         bool flush_result;
1698         unsigned long flags;
1699
1700         if (!ufshcd_is_clkgating_allowed(hba) ||
1701             !hba->clk_gating.is_initialized)
1702                 goto out;
1703         spin_lock_irqsave(hba->host->host_lock, flags);
1704         hba->clk_gating.active_reqs++;
1705
1706 start:
1707         switch (hba->clk_gating.state) {
1708         case CLKS_ON:
1709                 /*
1710                  * Wait for the ungate work to complete if in progress.
1711                  * Though the clocks may be in ON state, the link could
1712                  * still be in hibner8 state if hibern8 is allowed
1713                  * during clock gating.
1714                  * Make sure we exit hibern8 state also in addition to
1715                  * clocks being ON.
1716                  */
1717                 if (ufshcd_can_hibern8_during_gating(hba) &&
1718                     ufshcd_is_link_hibern8(hba)) {
1719                         if (async) {
1720                                 rc = -EAGAIN;
1721                                 hba->clk_gating.active_reqs--;
1722                                 break;
1723                         }
1724                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1725                         flush_result = flush_work(&hba->clk_gating.ungate_work);
1726                         if (hba->clk_gating.is_suspended && !flush_result)
1727                                 goto out;
1728                         spin_lock_irqsave(hba->host->host_lock, flags);
1729                         goto start;
1730                 }
1731                 break;
1732         case REQ_CLKS_OFF:
1733                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1734                         hba->clk_gating.state = CLKS_ON;
1735                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1736                                                 hba->clk_gating.state);
1737                         break;
1738                 }
1739                 /*
1740                  * If we are here, it means gating work is either done or
1741                  * currently running. Hence, fall through to cancel gating
1742                  * work and to enable clocks.
1743                  */
1744                 fallthrough;
1745         case CLKS_OFF:
1746                 hba->clk_gating.state = REQ_CLKS_ON;
1747                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1748                                         hba->clk_gating.state);
1749                 if (queue_work(hba->clk_gating.clk_gating_workq,
1750                                &hba->clk_gating.ungate_work))
1751                         ufshcd_scsi_block_requests(hba);
1752                 /*
1753                  * fall through to check if we should wait for this
1754                  * work to be done or not.
1755                  */
1756                 fallthrough;
1757         case REQ_CLKS_ON:
1758                 if (async) {
1759                         rc = -EAGAIN;
1760                         hba->clk_gating.active_reqs--;
1761                         break;
1762                 }
1763
1764                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1765                 flush_work(&hba->clk_gating.ungate_work);
1766                 /* Make sure state is CLKS_ON before returning */
1767                 spin_lock_irqsave(hba->host->host_lock, flags);
1768                 goto start;
1769         default:
1770                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1771                                 __func__, hba->clk_gating.state);
1772                 break;
1773         }
1774         spin_unlock_irqrestore(hba->host->host_lock, flags);
1775 out:
1776         return rc;
1777 }
1778 EXPORT_SYMBOL_GPL(ufshcd_hold);
1779
1780 static void ufshcd_gate_work(struct work_struct *work)
1781 {
1782         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1783                         clk_gating.gate_work.work);
1784         unsigned long flags;
1785         int ret;
1786
1787         spin_lock_irqsave(hba->host->host_lock, flags);
1788         /*
1789          * In case you are here to cancel this work the gating state
1790          * would be marked as REQ_CLKS_ON. In this case save time by
1791          * skipping the gating work and exit after changing the clock
1792          * state to CLKS_ON.
1793          */
1794         if (hba->clk_gating.is_suspended ||
1795                 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1796                 hba->clk_gating.state = CLKS_ON;
1797                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1798                                         hba->clk_gating.state);
1799                 goto rel_lock;
1800         }
1801
1802         if (hba->clk_gating.active_reqs
1803                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1804                 || hba->outstanding_reqs || hba->outstanding_tasks
1805                 || hba->active_uic_cmd || hba->uic_async_done)
1806                 goto rel_lock;
1807
1808         spin_unlock_irqrestore(hba->host->host_lock, flags);
1809
1810         /* put the link into hibern8 mode before turning off clocks */
1811         if (ufshcd_can_hibern8_during_gating(hba)) {
1812                 ret = ufshcd_uic_hibern8_enter(hba);
1813                 if (ret) {
1814                         hba->clk_gating.state = CLKS_ON;
1815                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1816                                         __func__, ret);
1817                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1818                                                 hba->clk_gating.state);
1819                         goto out;
1820                 }
1821                 ufshcd_set_link_hibern8(hba);
1822         }
1823
1824         ufshcd_disable_irq(hba);
1825
1826         ufshcd_setup_clocks(hba, false);
1827
1828         /* Put the host controller in low power mode if possible */
1829         ufshcd_hba_vreg_set_lpm(hba);
1830         /*
1831          * In case you are here to cancel this work the gating state
1832          * would be marked as REQ_CLKS_ON. In this case keep the state
1833          * as REQ_CLKS_ON which would anyway imply that clocks are off
1834          * and a request to turn them on is pending. By doing this way,
1835          * we keep the state machine in tact and this would ultimately
1836          * prevent from doing cancel work multiple times when there are
1837          * new requests arriving before the current cancel work is done.
1838          */
1839         spin_lock_irqsave(hba->host->host_lock, flags);
1840         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1841                 hba->clk_gating.state = CLKS_OFF;
1842                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1843                                         hba->clk_gating.state);
1844         }
1845 rel_lock:
1846         spin_unlock_irqrestore(hba->host->host_lock, flags);
1847 out:
1848         return;
1849 }
1850
1851 /* host lock must be held before calling this variant */
1852 static void __ufshcd_release(struct ufs_hba *hba)
1853 {
1854         if (!ufshcd_is_clkgating_allowed(hba))
1855                 return;
1856
1857         hba->clk_gating.active_reqs--;
1858
1859         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1860             hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1861             hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
1862             hba->active_uic_cmd || hba->uic_async_done ||
1863             hba->clk_gating.state == CLKS_OFF)
1864                 return;
1865
1866         hba->clk_gating.state = REQ_CLKS_OFF;
1867         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1868         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1869                            &hba->clk_gating.gate_work,
1870                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1871 }
1872
1873 void ufshcd_release(struct ufs_hba *hba)
1874 {
1875         unsigned long flags;
1876
1877         spin_lock_irqsave(hba->host->host_lock, flags);
1878         __ufshcd_release(hba);
1879         spin_unlock_irqrestore(hba->host->host_lock, flags);
1880 }
1881 EXPORT_SYMBOL_GPL(ufshcd_release);
1882
1883 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1884                 struct device_attribute *attr, char *buf)
1885 {
1886         struct ufs_hba *hba = dev_get_drvdata(dev);
1887
1888         return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1889 }
1890
1891 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
1892 {
1893         struct ufs_hba *hba = dev_get_drvdata(dev);
1894         unsigned long flags;
1895
1896         spin_lock_irqsave(hba->host->host_lock, flags);
1897         hba->clk_gating.delay_ms = value;
1898         spin_unlock_irqrestore(hba->host->host_lock, flags);
1899 }
1900 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
1901
1902 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1903                 struct device_attribute *attr, const char *buf, size_t count)
1904 {
1905         unsigned long value;
1906
1907         if (kstrtoul(buf, 0, &value))
1908                 return -EINVAL;
1909
1910         ufshcd_clkgate_delay_set(dev, value);
1911         return count;
1912 }
1913
1914 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1915                 struct device_attribute *attr, char *buf)
1916 {
1917         struct ufs_hba *hba = dev_get_drvdata(dev);
1918
1919         return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
1920 }
1921
1922 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1923                 struct device_attribute *attr, const char *buf, size_t count)
1924 {
1925         struct ufs_hba *hba = dev_get_drvdata(dev);
1926         unsigned long flags;
1927         u32 value;
1928
1929         if (kstrtou32(buf, 0, &value))
1930                 return -EINVAL;
1931
1932         value = !!value;
1933
1934         spin_lock_irqsave(hba->host->host_lock, flags);
1935         if (value == hba->clk_gating.is_enabled)
1936                 goto out;
1937
1938         if (value)
1939                 __ufshcd_release(hba);
1940         else
1941                 hba->clk_gating.active_reqs++;
1942
1943         hba->clk_gating.is_enabled = value;
1944 out:
1945         spin_unlock_irqrestore(hba->host->host_lock, flags);
1946         return count;
1947 }
1948
1949 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
1950 {
1951         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1952         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1953         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1954         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1955         hba->clk_gating.delay_attr.attr.mode = 0644;
1956         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1957                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1958
1959         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1960         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1961         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1962         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1963         hba->clk_gating.enable_attr.attr.mode = 0644;
1964         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1965                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1966 }
1967
1968 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
1969 {
1970         if (hba->clk_gating.delay_attr.attr.name)
1971                 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1972         if (hba->clk_gating.enable_attr.attr.name)
1973                 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1974 }
1975
1976 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1977 {
1978         char wq_name[sizeof("ufs_clk_gating_00")];
1979
1980         if (!ufshcd_is_clkgating_allowed(hba))
1981                 return;
1982
1983         hba->clk_gating.state = CLKS_ON;
1984
1985         hba->clk_gating.delay_ms = 150;
1986         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1987         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1988
1989         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1990                  hba->host->host_no);
1991         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1992                                         WQ_MEM_RECLAIM | WQ_HIGHPRI);
1993
1994         ufshcd_init_clk_gating_sysfs(hba);
1995
1996         hba->clk_gating.is_enabled = true;
1997         hba->clk_gating.is_initialized = true;
1998 }
1999
2000 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2001 {
2002         if (!hba->clk_gating.is_initialized)
2003                 return;
2004
2005         ufshcd_remove_clk_gating_sysfs(hba);
2006
2007         /* Ungate the clock if necessary. */
2008         ufshcd_hold(hba, false);
2009         hba->clk_gating.is_initialized = false;
2010         ufshcd_release(hba);
2011
2012         destroy_workqueue(hba->clk_gating.clk_gating_workq);
2013 }
2014
2015 /* Must be called with host lock acquired */
2016 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2017 {
2018         bool queue_resume_work = false;
2019         ktime_t curr_t = ktime_get();
2020         unsigned long flags;
2021
2022         if (!ufshcd_is_clkscaling_supported(hba))
2023                 return;
2024
2025         spin_lock_irqsave(hba->host->host_lock, flags);
2026         if (!hba->clk_scaling.active_reqs++)
2027                 queue_resume_work = true;
2028
2029         if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2030                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2031                 return;
2032         }
2033
2034         if (queue_resume_work)
2035                 queue_work(hba->clk_scaling.workq,
2036                            &hba->clk_scaling.resume_work);
2037
2038         if (!hba->clk_scaling.window_start_t) {
2039                 hba->clk_scaling.window_start_t = curr_t;
2040                 hba->clk_scaling.tot_busy_t = 0;
2041                 hba->clk_scaling.is_busy_started = false;
2042         }
2043
2044         if (!hba->clk_scaling.is_busy_started) {
2045                 hba->clk_scaling.busy_start_t = curr_t;
2046                 hba->clk_scaling.is_busy_started = true;
2047         }
2048         spin_unlock_irqrestore(hba->host->host_lock, flags);
2049 }
2050
2051 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2052 {
2053         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2054         unsigned long flags;
2055
2056         if (!ufshcd_is_clkscaling_supported(hba))
2057                 return;
2058
2059         spin_lock_irqsave(hba->host->host_lock, flags);
2060         hba->clk_scaling.active_reqs--;
2061         if (!hba->outstanding_reqs && scaling->is_busy_started) {
2062                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2063                                         scaling->busy_start_t));
2064                 scaling->busy_start_t = 0;
2065                 scaling->is_busy_started = false;
2066         }
2067         spin_unlock_irqrestore(hba->host->host_lock, flags);
2068 }
2069
2070 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2071 {
2072         if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2073                 return READ;
2074         else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2075                 return WRITE;
2076         else
2077                 return -EINVAL;
2078 }
2079
2080 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2081                                                 struct ufshcd_lrb *lrbp)
2082 {
2083         struct ufs_hba_monitor *m = &hba->monitor;
2084
2085         return (m->enabled && lrbp && lrbp->cmd &&
2086                 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2087                 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2088 }
2089
2090 static void ufshcd_start_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2091 {
2092         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2093         unsigned long flags;
2094
2095         spin_lock_irqsave(hba->host->host_lock, flags);
2096         if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2097                 hba->monitor.busy_start_ts[dir] = ktime_get();
2098         spin_unlock_irqrestore(hba->host->host_lock, flags);
2099 }
2100
2101 static void ufshcd_update_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2102 {
2103         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2104         unsigned long flags;
2105
2106         spin_lock_irqsave(hba->host->host_lock, flags);
2107         if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2108                 struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2109                 struct ufs_hba_monitor *m = &hba->monitor;
2110                 ktime_t now, inc, lat;
2111
2112                 now = lrbp->compl_time_stamp;
2113                 inc = ktime_sub(now, m->busy_start_ts[dir]);
2114                 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2115                 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2116
2117                 /* Update latencies */
2118                 m->nr_req[dir]++;
2119                 lat = ktime_sub(now, lrbp->issue_time_stamp);
2120                 m->lat_sum[dir] += lat;
2121                 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2122                         m->lat_max[dir] = lat;
2123                 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2124                         m->lat_min[dir] = lat;
2125
2126                 m->nr_queued[dir]--;
2127                 /* Push forward the busy start of monitor */
2128                 m->busy_start_ts[dir] = now;
2129         }
2130         spin_unlock_irqrestore(hba->host->host_lock, flags);
2131 }
2132
2133 /**
2134  * ufshcd_send_command - Send SCSI or device management commands
2135  * @hba: per adapter instance
2136  * @task_tag: Task tag of the command
2137  */
2138 static inline
2139 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2140 {
2141         struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2142         unsigned long flags;
2143
2144         lrbp->issue_time_stamp = ktime_get();
2145         lrbp->compl_time_stamp = ktime_set(0, 0);
2146         ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2147         ufshcd_clk_scaling_start_busy(hba);
2148         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2149                 ufshcd_start_monitor(hba, lrbp);
2150
2151         spin_lock_irqsave(&hba->outstanding_lock, flags);
2152         if (hba->vops && hba->vops->setup_xfer_req)
2153                 hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd);
2154         __set_bit(task_tag, &hba->outstanding_reqs);
2155         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
2156         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2157 }
2158
2159 /**
2160  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2161  * @lrbp: pointer to local reference block
2162  */
2163 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2164 {
2165         u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2166         int len;
2167
2168         if (sense_buffer &&
2169             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
2170                 int len_to_copy;
2171
2172                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2173                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2174
2175                 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2176                        len_to_copy);
2177         }
2178 }
2179
2180 /**
2181  * ufshcd_copy_query_response() - Copy the Query Response and the data
2182  * descriptor
2183  * @hba: per adapter instance
2184  * @lrbp: pointer to local reference block
2185  */
2186 static
2187 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2188 {
2189         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2190
2191         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2192
2193         /* Get the descriptor */
2194         if (hba->dev_cmd.query.descriptor &&
2195             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2196                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2197                                 GENERAL_UPIU_REQUEST_SIZE;
2198                 u16 resp_len;
2199                 u16 buf_len;
2200
2201                 /* data segment length */
2202                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2203                                                 MASK_QUERY_DATA_SEG_LEN;
2204                 buf_len = be16_to_cpu(
2205                                 hba->dev_cmd.query.request.upiu_req.length);
2206                 if (likely(buf_len >= resp_len)) {
2207                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2208                 } else {
2209                         dev_warn(hba->dev,
2210                                  "%s: rsp size %d is bigger than buffer size %d",
2211                                  __func__, resp_len, buf_len);
2212                         return -EINVAL;
2213                 }
2214         }
2215
2216         return 0;
2217 }
2218
2219 /**
2220  * ufshcd_hba_capabilities - Read controller capabilities
2221  * @hba: per adapter instance
2222  *
2223  * Return: 0 on success, negative on error.
2224  */
2225 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2226 {
2227         int err;
2228
2229         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2230
2231         /* nutrs and nutmrs are 0 based values */
2232         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2233         hba->nutmrs =
2234         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2235         hba->reserved_slot = hba->nutrs - 1;
2236
2237         /* Read crypto capabilities */
2238         err = ufshcd_hba_init_crypto_capabilities(hba);
2239         if (err)
2240                 dev_err(hba->dev, "crypto setup failed\n");
2241
2242         return err;
2243 }
2244
2245 /**
2246  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2247  *                            to accept UIC commands
2248  * @hba: per adapter instance
2249  * Return true on success, else false
2250  */
2251 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2252 {
2253         return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY;
2254 }
2255
2256 /**
2257  * ufshcd_get_upmcrs - Get the power mode change request status
2258  * @hba: Pointer to adapter instance
2259  *
2260  * This function gets the UPMCRS field of HCS register
2261  * Returns value of UPMCRS field
2262  */
2263 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2264 {
2265         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2266 }
2267
2268 /**
2269  * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2270  * @hba: per adapter instance
2271  * @uic_cmd: UIC command
2272  */
2273 static inline void
2274 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2275 {
2276         lockdep_assert_held(&hba->uic_cmd_mutex);
2277
2278         WARN_ON(hba->active_uic_cmd);
2279
2280         hba->active_uic_cmd = uic_cmd;
2281
2282         /* Write Args */
2283         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2284         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2285         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2286
2287         ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2288
2289         /* Write UIC Cmd */
2290         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2291                       REG_UIC_COMMAND);
2292 }
2293
2294 /**
2295  * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2296  * @hba: per adapter instance
2297  * @uic_cmd: UIC command
2298  *
2299  * Returns 0 only if success.
2300  */
2301 static int
2302 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2303 {
2304         int ret;
2305         unsigned long flags;
2306
2307         lockdep_assert_held(&hba->uic_cmd_mutex);
2308
2309         if (wait_for_completion_timeout(&uic_cmd->done,
2310                                         msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2311                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2312         } else {
2313                 ret = -ETIMEDOUT;
2314                 dev_err(hba->dev,
2315                         "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2316                         uic_cmd->command, uic_cmd->argument3);
2317
2318                 if (!uic_cmd->cmd_active) {
2319                         dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2320                                 __func__);
2321                         ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2322                 }
2323         }
2324
2325         spin_lock_irqsave(hba->host->host_lock, flags);
2326         hba->active_uic_cmd = NULL;
2327         spin_unlock_irqrestore(hba->host->host_lock, flags);
2328
2329         return ret;
2330 }
2331
2332 /**
2333  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2334  * @hba: per adapter instance
2335  * @uic_cmd: UIC command
2336  * @completion: initialize the completion only if this is set to true
2337  *
2338  * Returns 0 only if success.
2339  */
2340 static int
2341 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2342                       bool completion)
2343 {
2344         lockdep_assert_held(&hba->uic_cmd_mutex);
2345         lockdep_assert_held(hba->host->host_lock);
2346
2347         if (!ufshcd_ready_for_uic_cmd(hba)) {
2348                 dev_err(hba->dev,
2349                         "Controller not ready to accept UIC commands\n");
2350                 return -EIO;
2351         }
2352
2353         if (completion)
2354                 init_completion(&uic_cmd->done);
2355
2356         uic_cmd->cmd_active = 1;
2357         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2358
2359         return 0;
2360 }
2361
2362 /**
2363  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2364  * @hba: per adapter instance
2365  * @uic_cmd: UIC command
2366  *
2367  * Returns 0 only if success.
2368  */
2369 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2370 {
2371         int ret;
2372         unsigned long flags;
2373
2374         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2375                 return 0;
2376
2377         ufshcd_hold(hba, false);
2378         mutex_lock(&hba->uic_cmd_mutex);
2379         ufshcd_add_delay_before_dme_cmd(hba);
2380
2381         spin_lock_irqsave(hba->host->host_lock, flags);
2382         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2383         spin_unlock_irqrestore(hba->host->host_lock, flags);
2384         if (!ret)
2385                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2386
2387         mutex_unlock(&hba->uic_cmd_mutex);
2388
2389         ufshcd_release(hba);
2390         return ret;
2391 }
2392
2393 /**
2394  * ufshcd_map_sg - Map scatter-gather list to prdt
2395  * @hba: per adapter instance
2396  * @lrbp: pointer to local reference block
2397  *
2398  * Returns 0 in case of success, non-zero value in case of failure
2399  */
2400 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2401 {
2402         struct ufshcd_sg_entry *prd_table;
2403         struct scatterlist *sg;
2404         struct scsi_cmnd *cmd;
2405         int sg_segments;
2406         int i;
2407
2408         cmd = lrbp->cmd;
2409         sg_segments = scsi_dma_map(cmd);
2410         if (sg_segments < 0)
2411                 return sg_segments;
2412
2413         if (sg_segments) {
2414
2415                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2416                         lrbp->utr_descriptor_ptr->prd_table_length =
2417                                 cpu_to_le16((sg_segments *
2418                                         sizeof(struct ufshcd_sg_entry)));
2419                 else
2420                         lrbp->utr_descriptor_ptr->prd_table_length =
2421                                 cpu_to_le16(sg_segments);
2422
2423                 prd_table = lrbp->ucd_prdt_ptr;
2424
2425                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2426                         const unsigned int len = sg_dma_len(sg);
2427
2428                         /*
2429                          * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2430                          * based value that indicates the length, in bytes, of
2431                          * the data block. A maximum of length of 256KB may
2432                          * exist for any entry. Bits 1:0 of this field shall be
2433                          * 11b to indicate Dword granularity. A value of '3'
2434                          * indicates 4 bytes, '7' indicates 8 bytes, etc."
2435                          */
2436                         WARN_ONCE(len > 256 * 1024, "len = %#x\n", len);
2437                         prd_table[i].size = cpu_to_le32(len - 1);
2438                         prd_table[i].addr = cpu_to_le64(sg->dma_address);
2439                         prd_table[i].reserved = 0;
2440                 }
2441         } else {
2442                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2443         }
2444
2445         return 0;
2446 }
2447
2448 /**
2449  * ufshcd_enable_intr - enable interrupts
2450  * @hba: per adapter instance
2451  * @intrs: interrupt bits
2452  */
2453 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2454 {
2455         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2456
2457         if (hba->ufs_version == ufshci_version(1, 0)) {
2458                 u32 rw;
2459                 rw = set & INTERRUPT_MASK_RW_VER_10;
2460                 set = rw | ((set ^ intrs) & intrs);
2461         } else {
2462                 set |= intrs;
2463         }
2464
2465         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2466 }
2467
2468 /**
2469  * ufshcd_disable_intr - disable interrupts
2470  * @hba: per adapter instance
2471  * @intrs: interrupt bits
2472  */
2473 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2474 {
2475         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2476
2477         if (hba->ufs_version == ufshci_version(1, 0)) {
2478                 u32 rw;
2479                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2480                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2481                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2482
2483         } else {
2484                 set &= ~intrs;
2485         }
2486
2487         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2488 }
2489
2490 /**
2491  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2492  * descriptor according to request
2493  * @lrbp: pointer to local reference block
2494  * @upiu_flags: flags required in the header
2495  * @cmd_dir: requests data direction
2496  */
2497 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2498                         u8 *upiu_flags, enum dma_data_direction cmd_dir)
2499 {
2500         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2501         u32 data_direction;
2502         u32 dword_0;
2503         u32 dword_1 = 0;
2504         u32 dword_3 = 0;
2505
2506         if (cmd_dir == DMA_FROM_DEVICE) {
2507                 data_direction = UTP_DEVICE_TO_HOST;
2508                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2509         } else if (cmd_dir == DMA_TO_DEVICE) {
2510                 data_direction = UTP_HOST_TO_DEVICE;
2511                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2512         } else {
2513                 data_direction = UTP_NO_DATA_TRANSFER;
2514                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2515         }
2516
2517         dword_0 = data_direction | (lrbp->command_type
2518                                 << UPIU_COMMAND_TYPE_OFFSET);
2519         if (lrbp->intr_cmd)
2520                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2521
2522         /* Prepare crypto related dwords */
2523         ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2524
2525         /* Transfer request descriptor header fields */
2526         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2527         req_desc->header.dword_1 = cpu_to_le32(dword_1);
2528         /*
2529          * assigning invalid value for command status. Controller
2530          * updates OCS on command completion, with the command
2531          * status
2532          */
2533         req_desc->header.dword_2 =
2534                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2535         req_desc->header.dword_3 = cpu_to_le32(dword_3);
2536
2537         req_desc->prd_table_length = 0;
2538 }
2539
2540 /**
2541  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2542  * for scsi commands
2543  * @lrbp: local reference block pointer
2544  * @upiu_flags: flags
2545  */
2546 static
2547 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2548 {
2549         struct scsi_cmnd *cmd = lrbp->cmd;
2550         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2551         unsigned short cdb_len;
2552
2553         /* command descriptor fields */
2554         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2555                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2556                                 lrbp->lun, lrbp->task_tag);
2557         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2558                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2559
2560         /* Total EHS length and Data segment length will be zero */
2561         ucd_req_ptr->header.dword_2 = 0;
2562
2563         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2564
2565         cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2566         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2567         memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2568
2569         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2570 }
2571
2572 /**
2573  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2574  * for query requsts
2575  * @hba: UFS hba
2576  * @lrbp: local reference block pointer
2577  * @upiu_flags: flags
2578  */
2579 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2580                                 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2581 {
2582         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2583         struct ufs_query *query = &hba->dev_cmd.query;
2584         u16 len = be16_to_cpu(query->request.upiu_req.length);
2585
2586         /* Query request header */
2587         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2588                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2589                         lrbp->lun, lrbp->task_tag);
2590         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2591                         0, query->request.query_func, 0, 0);
2592
2593         /* Data segment length only need for WRITE_DESC */
2594         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2595                 ucd_req_ptr->header.dword_2 =
2596                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2597         else
2598                 ucd_req_ptr->header.dword_2 = 0;
2599
2600         /* Copy the Query Request buffer as is */
2601         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2602                         QUERY_OSF_SIZE);
2603
2604         /* Copy the Descriptor */
2605         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2606                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2607
2608         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2609 }
2610
2611 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2612 {
2613         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2614
2615         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2616
2617         /* command descriptor fields */
2618         ucd_req_ptr->header.dword_0 =
2619                 UPIU_HEADER_DWORD(
2620                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2621         /* clear rest of the fields of basic header */
2622         ucd_req_ptr->header.dword_1 = 0;
2623         ucd_req_ptr->header.dword_2 = 0;
2624
2625         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2626 }
2627
2628 /**
2629  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2630  *                           for Device Management Purposes
2631  * @hba: per adapter instance
2632  * @lrbp: pointer to local reference block
2633  */
2634 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2635                                       struct ufshcd_lrb *lrbp)
2636 {
2637         u8 upiu_flags;
2638         int ret = 0;
2639
2640         if (hba->ufs_version <= ufshci_version(1, 1))
2641                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2642         else
2643                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2644
2645         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2646         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2647                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2648         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2649                 ufshcd_prepare_utp_nop_upiu(lrbp);
2650         else
2651                 ret = -EINVAL;
2652
2653         return ret;
2654 }
2655
2656 /**
2657  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2658  *                         for SCSI Purposes
2659  * @hba: per adapter instance
2660  * @lrbp: pointer to local reference block
2661  */
2662 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2663 {
2664         u8 upiu_flags;
2665         int ret = 0;
2666
2667         if (hba->ufs_version <= ufshci_version(1, 1))
2668                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2669         else
2670                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2671
2672         if (likely(lrbp->cmd)) {
2673                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2674                                                 lrbp->cmd->sc_data_direction);
2675                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2676         } else {
2677                 ret = -EINVAL;
2678         }
2679
2680         return ret;
2681 }
2682
2683 /**
2684  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2685  * @upiu_wlun_id: UPIU W-LUN id
2686  *
2687  * Returns SCSI W-LUN id
2688  */
2689 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2690 {
2691         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2692 }
2693
2694 static inline bool is_device_wlun(struct scsi_device *sdev)
2695 {
2696         return sdev->lun ==
2697                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2698 }
2699
2700 /*
2701  * Associate the UFS controller queue with the default and poll HCTX types.
2702  * Initialize the mq_map[] arrays.
2703  */
2704 static int ufshcd_map_queues(struct Scsi_Host *shost)
2705 {
2706         int i, ret;
2707
2708         for (i = 0; i < shost->nr_maps; i++) {
2709                 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2710
2711                 switch (i) {
2712                 case HCTX_TYPE_DEFAULT:
2713                 case HCTX_TYPE_POLL:
2714                         map->nr_queues = 1;
2715                         break;
2716                 case HCTX_TYPE_READ:
2717                         map->nr_queues = 0;
2718                         continue;
2719                 default:
2720                         WARN_ON_ONCE(true);
2721                 }
2722                 map->queue_offset = 0;
2723                 ret = blk_mq_map_queues(map);
2724                 WARN_ON_ONCE(ret);
2725         }
2726
2727         return 0;
2728 }
2729
2730 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2731 {
2732         struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2733         struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2734         dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2735                 i * sizeof(struct utp_transfer_cmd_desc);
2736         u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2737                                        response_upiu);
2738         u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2739
2740         lrb->utr_descriptor_ptr = utrdlp + i;
2741         lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2742                 i * sizeof(struct utp_transfer_req_desc);
2743         lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2744         lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2745         lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2746         lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2747         lrb->ucd_prdt_ptr = cmd_descp[i].prd_table;
2748         lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2749 }
2750
2751 /**
2752  * ufshcd_queuecommand - main entry point for SCSI requests
2753  * @host: SCSI host pointer
2754  * @cmd: command from SCSI Midlayer
2755  *
2756  * Returns 0 for success, non-zero in case of failure
2757  */
2758 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2759 {
2760         struct ufs_hba *hba = shost_priv(host);
2761         int tag = scsi_cmd_to_rq(cmd)->tag;
2762         struct ufshcd_lrb *lrbp;
2763         int err = 0;
2764
2765         WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag);
2766
2767         /*
2768          * Allows the UFS error handler to wait for prior ufshcd_queuecommand()
2769          * calls.
2770          */
2771         rcu_read_lock();
2772
2773         switch (hba->ufshcd_state) {
2774         case UFSHCD_STATE_OPERATIONAL:
2775                 break;
2776         case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2777                 /*
2778                  * SCSI error handler can call ->queuecommand() while UFS error
2779                  * handler is in progress. Error interrupts could change the
2780                  * state from UFSHCD_STATE_RESET to
2781                  * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2782                  * being issued in that case.
2783                  */
2784                 if (ufshcd_eh_in_progress(hba)) {
2785                         err = SCSI_MLQUEUE_HOST_BUSY;
2786                         goto out;
2787                 }
2788                 break;
2789         case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2790                 /*
2791                  * pm_runtime_get_sync() is used at error handling preparation
2792                  * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2793                  * PM ops, it can never be finished if we let SCSI layer keep
2794                  * retrying it, which gets err handler stuck forever. Neither
2795                  * can we let the scsi cmd pass through, because UFS is in bad
2796                  * state, the scsi cmd may eventually time out, which will get
2797                  * err handler blocked for too long. So, just fail the scsi cmd
2798                  * sent from PM ops, err handler can recover PM error anyways.
2799                  */
2800                 if (hba->pm_op_in_progress) {
2801                         hba->force_reset = true;
2802                         set_host_byte(cmd, DID_BAD_TARGET);
2803                         scsi_done(cmd);
2804                         goto out;
2805                 }
2806                 fallthrough;
2807         case UFSHCD_STATE_RESET:
2808                 err = SCSI_MLQUEUE_HOST_BUSY;
2809                 goto out;
2810         case UFSHCD_STATE_ERROR:
2811                 set_host_byte(cmd, DID_ERROR);
2812                 scsi_done(cmd);
2813                 goto out;
2814         }
2815
2816         hba->req_abort_count = 0;
2817
2818         err = ufshcd_hold(hba, true);
2819         if (err) {
2820                 err = SCSI_MLQUEUE_HOST_BUSY;
2821                 goto out;
2822         }
2823         WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2824                 (hba->clk_gating.state != CLKS_ON));
2825
2826         lrbp = &hba->lrb[tag];
2827         WARN_ON(lrbp->cmd);
2828         lrbp->cmd = cmd;
2829         lrbp->task_tag = tag;
2830         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2831         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
2832
2833         ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
2834
2835         lrbp->req_abort_skip = false;
2836
2837         ufshpb_prep(hba, lrbp);
2838
2839         ufshcd_comp_scsi_upiu(hba, lrbp);
2840
2841         err = ufshcd_map_sg(hba, lrbp);
2842         if (err) {
2843                 lrbp->cmd = NULL;
2844                 ufshcd_release(hba);
2845                 goto out;
2846         }
2847
2848         ufshcd_send_command(hba, tag);
2849
2850 out:
2851         rcu_read_unlock();
2852
2853         if (ufs_trigger_eh()) {
2854                 unsigned long flags;
2855
2856                 spin_lock_irqsave(hba->host->host_lock, flags);
2857                 ufshcd_schedule_eh_work(hba);
2858                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2859         }
2860
2861         return err;
2862 }
2863
2864 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2865                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2866 {
2867         lrbp->cmd = NULL;
2868         lrbp->task_tag = tag;
2869         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2870         lrbp->intr_cmd = true; /* No interrupt aggregation */
2871         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2872         hba->dev_cmd.type = cmd_type;
2873
2874         return ufshcd_compose_devman_upiu(hba, lrbp);
2875 }
2876
2877 /*
2878  * Clear all the requests from the controller for which a bit has been set in
2879  * @mask and wait until the controller confirms that these requests have been
2880  * cleared.
2881  */
2882 static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask)
2883 {
2884         unsigned long flags;
2885
2886         /* clear outstanding transaction before retry */
2887         spin_lock_irqsave(hba->host->host_lock, flags);
2888         ufshcd_utrl_clear(hba, mask);
2889         spin_unlock_irqrestore(hba->host->host_lock, flags);
2890
2891         /*
2892          * wait for h/w to clear corresponding bit in door-bell.
2893          * max. wait is 1 sec.
2894          */
2895         return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
2896                                         mask, ~mask, 1000, 1000);
2897 }
2898
2899 static int
2900 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2901 {
2902         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2903
2904         /* Get the UPIU response */
2905         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2906                                 UPIU_RSP_CODE_OFFSET;
2907         return query_res->response;
2908 }
2909
2910 /**
2911  * ufshcd_dev_cmd_completion() - handles device management command responses
2912  * @hba: per adapter instance
2913  * @lrbp: pointer to local reference block
2914  */
2915 static int
2916 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2917 {
2918         int resp;
2919         int err = 0;
2920
2921         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2922         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2923
2924         switch (resp) {
2925         case UPIU_TRANSACTION_NOP_IN:
2926                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2927                         err = -EINVAL;
2928                         dev_err(hba->dev, "%s: unexpected response %x\n",
2929                                         __func__, resp);
2930                 }
2931                 break;
2932         case UPIU_TRANSACTION_QUERY_RSP:
2933                 err = ufshcd_check_query_response(hba, lrbp);
2934                 if (!err)
2935                         err = ufshcd_copy_query_response(hba, lrbp);
2936                 break;
2937         case UPIU_TRANSACTION_REJECT_UPIU:
2938                 /* TODO: handle Reject UPIU Response */
2939                 err = -EPERM;
2940                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2941                                 __func__);
2942                 break;
2943         default:
2944                 err = -EINVAL;
2945                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2946                                 __func__, resp);
2947                 break;
2948         }
2949
2950         return err;
2951 }
2952
2953 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2954                 struct ufshcd_lrb *lrbp, int max_timeout)
2955 {
2956         unsigned long time_left = msecs_to_jiffies(max_timeout);
2957         unsigned long flags;
2958         bool pending;
2959         int err;
2960
2961 retry:
2962         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2963                                                 time_left);
2964
2965         if (likely(time_left)) {
2966                 /*
2967                  * The completion handler called complete() and the caller of
2968                  * this function still owns the @lrbp tag so the code below does
2969                  * not trigger any race conditions.
2970                  */
2971                 hba->dev_cmd.complete = NULL;
2972                 err = ufshcd_get_tr_ocs(lrbp);
2973                 if (!err)
2974                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2975         } else {
2976                 err = -ETIMEDOUT;
2977                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2978                         __func__, lrbp->task_tag);
2979                 if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
2980                         /* successfully cleared the command, retry if needed */
2981                         err = -EAGAIN;
2982                         /*
2983                          * Since clearing the command succeeded we also need to
2984                          * clear the task tag bit from the outstanding_reqs
2985                          * variable.
2986                          */
2987                         spin_lock_irqsave(&hba->outstanding_lock, flags);
2988                         pending = test_bit(lrbp->task_tag,
2989                                            &hba->outstanding_reqs);
2990                         if (pending) {
2991                                 hba->dev_cmd.complete = NULL;
2992                                 __clear_bit(lrbp->task_tag,
2993                                             &hba->outstanding_reqs);
2994                         }
2995                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2996
2997                         if (!pending) {
2998                                 /*
2999                                  * The completion handler ran while we tried to
3000                                  * clear the command.
3001                                  */
3002                                 time_left = 1;
3003                                 goto retry;
3004                         }
3005                 } else {
3006                         dev_err(hba->dev, "%s: failed to clear tag %d\n",
3007                                 __func__, lrbp->task_tag);
3008                 }
3009         }
3010
3011         return err;
3012 }
3013
3014 /**
3015  * ufshcd_exec_dev_cmd - API for sending device management requests
3016  * @hba: UFS hba
3017  * @cmd_type: specifies the type (NOP, Query...)
3018  * @timeout: timeout in milliseconds
3019  *
3020  * NOTE: Since there is only one available tag for device management commands,
3021  * it is expected you hold the hba->dev_cmd.lock mutex.
3022  */
3023 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3024                 enum dev_cmd_type cmd_type, int timeout)
3025 {
3026         DECLARE_COMPLETION_ONSTACK(wait);
3027         const u32 tag = hba->reserved_slot;
3028         struct ufshcd_lrb *lrbp;
3029         int err;
3030
3031         /* Protects use of hba->reserved_slot. */
3032         lockdep_assert_held(&hba->dev_cmd.lock);
3033
3034         down_read(&hba->clk_scaling_lock);
3035
3036         lrbp = &hba->lrb[tag];
3037         WARN_ON(lrbp->cmd);
3038         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3039         if (unlikely(err))
3040                 goto out;
3041
3042         hba->dev_cmd.complete = &wait;
3043
3044         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3045
3046         ufshcd_send_command(hba, tag);
3047         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3048         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3049                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3050
3051 out:
3052         up_read(&hba->clk_scaling_lock);
3053         return err;
3054 }
3055
3056 /**
3057  * ufshcd_init_query() - init the query response and request parameters
3058  * @hba: per-adapter instance
3059  * @request: address of the request pointer to be initialized
3060  * @response: address of the response pointer to be initialized
3061  * @opcode: operation to perform
3062  * @idn: flag idn to access
3063  * @index: LU number to access
3064  * @selector: query/flag/descriptor further identification
3065  */
3066 static inline void ufshcd_init_query(struct ufs_hba *hba,
3067                 struct ufs_query_req **request, struct ufs_query_res **response,
3068                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3069 {
3070         *request = &hba->dev_cmd.query.request;
3071         *response = &hba->dev_cmd.query.response;
3072         memset(*request, 0, sizeof(struct ufs_query_req));
3073         memset(*response, 0, sizeof(struct ufs_query_res));
3074         (*request)->upiu_req.opcode = opcode;
3075         (*request)->upiu_req.idn = idn;
3076         (*request)->upiu_req.index = index;
3077         (*request)->upiu_req.selector = selector;
3078 }
3079
3080 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3081         enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3082 {
3083         int ret;
3084         int retries;
3085
3086         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3087                 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3088                 if (ret)
3089                         dev_dbg(hba->dev,
3090                                 "%s: failed with error %d, retries %d\n",
3091                                 __func__, ret, retries);
3092                 else
3093                         break;
3094         }
3095
3096         if (ret)
3097                 dev_err(hba->dev,
3098                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
3099                         __func__, opcode, idn, ret, retries);
3100         return ret;
3101 }
3102
3103 /**
3104  * ufshcd_query_flag() - API function for sending flag query requests
3105  * @hba: per-adapter instance
3106  * @opcode: flag query to perform
3107  * @idn: flag idn to access
3108  * @index: flag index to access
3109  * @flag_res: the flag value after the query request completes
3110  *
3111  * Returns 0 for success, non-zero in case of failure
3112  */
3113 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3114                         enum flag_idn idn, u8 index, bool *flag_res)
3115 {
3116         struct ufs_query_req *request = NULL;
3117         struct ufs_query_res *response = NULL;
3118         int err, selector = 0;
3119         int timeout = QUERY_REQ_TIMEOUT;
3120
3121         BUG_ON(!hba);
3122
3123         ufshcd_hold(hba, false);
3124         mutex_lock(&hba->dev_cmd.lock);
3125         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3126                         selector);
3127
3128         switch (opcode) {
3129         case UPIU_QUERY_OPCODE_SET_FLAG:
3130         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3131         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3132                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3133                 break;
3134         case UPIU_QUERY_OPCODE_READ_FLAG:
3135                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3136                 if (!flag_res) {
3137                         /* No dummy reads */
3138                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
3139                                         __func__);
3140                         err = -EINVAL;
3141                         goto out_unlock;
3142                 }
3143                 break;
3144         default:
3145                 dev_err(hba->dev,
3146                         "%s: Expected query flag opcode but got = %d\n",
3147                         __func__, opcode);
3148                 err = -EINVAL;
3149                 goto out_unlock;
3150         }
3151
3152         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3153
3154         if (err) {
3155                 dev_err(hba->dev,
3156                         "%s: Sending flag query for idn %d failed, err = %d\n",
3157                         __func__, idn, err);
3158                 goto out_unlock;
3159         }
3160
3161         if (flag_res)
3162                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3163                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3164
3165 out_unlock:
3166         mutex_unlock(&hba->dev_cmd.lock);
3167         ufshcd_release(hba);
3168         return err;
3169 }
3170
3171 /**
3172  * ufshcd_query_attr - API function for sending attribute requests
3173  * @hba: per-adapter instance
3174  * @opcode: attribute opcode
3175  * @idn: attribute idn to access
3176  * @index: index field
3177  * @selector: selector field
3178  * @attr_val: the attribute value after the query request completes
3179  *
3180  * Returns 0 for success, non-zero in case of failure
3181 */
3182 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3183                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3184 {
3185         struct ufs_query_req *request = NULL;
3186         struct ufs_query_res *response = NULL;
3187         int err;
3188
3189         BUG_ON(!hba);
3190
3191         if (!attr_val) {
3192                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3193                                 __func__, opcode);
3194                 return -EINVAL;
3195         }
3196
3197         ufshcd_hold(hba, false);
3198
3199         mutex_lock(&hba->dev_cmd.lock);
3200         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3201                         selector);
3202
3203         switch (opcode) {
3204         case UPIU_QUERY_OPCODE_WRITE_ATTR:
3205                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3206                 request->upiu_req.value = cpu_to_be32(*attr_val);
3207                 break;
3208         case UPIU_QUERY_OPCODE_READ_ATTR:
3209                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3210                 break;
3211         default:
3212                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3213                                 __func__, opcode);
3214                 err = -EINVAL;
3215                 goto out_unlock;
3216         }
3217
3218         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3219
3220         if (err) {
3221                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3222                                 __func__, opcode, idn, index, err);
3223                 goto out_unlock;
3224         }
3225
3226         *attr_val = be32_to_cpu(response->upiu_res.value);
3227
3228 out_unlock:
3229         mutex_unlock(&hba->dev_cmd.lock);
3230         ufshcd_release(hba);
3231         return err;
3232 }
3233
3234 /**
3235  * ufshcd_query_attr_retry() - API function for sending query
3236  * attribute with retries
3237  * @hba: per-adapter instance
3238  * @opcode: attribute opcode
3239  * @idn: attribute idn to access
3240  * @index: index field
3241  * @selector: selector field
3242  * @attr_val: the attribute value after the query request
3243  * completes
3244  *
3245  * Returns 0 for success, non-zero in case of failure
3246 */
3247 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3248         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3249         u32 *attr_val)
3250 {
3251         int ret = 0;
3252         u32 retries;
3253
3254         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3255                 ret = ufshcd_query_attr(hba, opcode, idn, index,
3256                                                 selector, attr_val);
3257                 if (ret)
3258                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3259                                 __func__, ret, retries);
3260                 else
3261                         break;
3262         }
3263
3264         if (ret)
3265                 dev_err(hba->dev,
3266                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3267                         __func__, idn, ret, QUERY_REQ_RETRIES);
3268         return ret;
3269 }
3270
3271 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3272                         enum query_opcode opcode, enum desc_idn idn, u8 index,
3273                         u8 selector, u8 *desc_buf, int *buf_len)
3274 {
3275         struct ufs_query_req *request = NULL;
3276         struct ufs_query_res *response = NULL;
3277         int err;
3278
3279         BUG_ON(!hba);
3280
3281         if (!desc_buf) {
3282                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3283                                 __func__, opcode);
3284                 return -EINVAL;
3285         }
3286
3287         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3288                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3289                                 __func__, *buf_len);
3290                 return -EINVAL;
3291         }
3292
3293         ufshcd_hold(hba, false);
3294
3295         mutex_lock(&hba->dev_cmd.lock);
3296         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3297                         selector);
3298         hba->dev_cmd.query.descriptor = desc_buf;
3299         request->upiu_req.length = cpu_to_be16(*buf_len);
3300
3301         switch (opcode) {
3302         case UPIU_QUERY_OPCODE_WRITE_DESC:
3303                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3304                 break;
3305         case UPIU_QUERY_OPCODE_READ_DESC:
3306                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3307                 break;
3308         default:
3309                 dev_err(hba->dev,
3310                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3311                                 __func__, opcode);
3312                 err = -EINVAL;
3313                 goto out_unlock;
3314         }
3315
3316         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3317
3318         if (err) {
3319                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3320                                 __func__, opcode, idn, index, err);
3321                 goto out_unlock;
3322         }
3323
3324         *buf_len = be16_to_cpu(response->upiu_res.length);
3325
3326 out_unlock:
3327         hba->dev_cmd.query.descriptor = NULL;
3328         mutex_unlock(&hba->dev_cmd.lock);
3329         ufshcd_release(hba);
3330         return err;
3331 }
3332
3333 /**
3334  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3335  * @hba: per-adapter instance
3336  * @opcode: attribute opcode
3337  * @idn: attribute idn to access
3338  * @index: index field
3339  * @selector: selector field
3340  * @desc_buf: the buffer that contains the descriptor
3341  * @buf_len: length parameter passed to the device
3342  *
3343  * Returns 0 for success, non-zero in case of failure.
3344  * The buf_len parameter will contain, on return, the length parameter
3345  * received on the response.
3346  */
3347 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3348                                   enum query_opcode opcode,
3349                                   enum desc_idn idn, u8 index,
3350                                   u8 selector,
3351                                   u8 *desc_buf, int *buf_len)
3352 {
3353         int err;
3354         int retries;
3355
3356         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3357                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3358                                                 selector, desc_buf, buf_len);
3359                 if (!err || err == -EINVAL)
3360                         break;
3361         }
3362
3363         return err;
3364 }
3365
3366 /**
3367  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3368  * @hba: Pointer to adapter instance
3369  * @desc_id: descriptor idn value
3370  * @desc_len: mapped desc length (out)
3371  */
3372 void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3373                                   int *desc_len)
3374 {
3375         if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3376             desc_id == QUERY_DESC_IDN_RFU_1)
3377                 *desc_len = 0;
3378         else
3379                 *desc_len = hba->desc_size[desc_id];
3380 }
3381 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3382
3383 static void ufshcd_update_desc_length(struct ufs_hba *hba,
3384                                       enum desc_idn desc_id, int desc_index,
3385                                       unsigned char desc_len)
3386 {
3387         if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3388             desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3389                 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3390                  * than the RPMB unit, however, both descriptors share the same
3391                  * desc_idn, to cover both unit descriptors with one length, we
3392                  * choose the normal unit descriptor length by desc_index.
3393                  */
3394                 hba->desc_size[desc_id] = desc_len;
3395 }
3396
3397 /**
3398  * ufshcd_read_desc_param - read the specified descriptor parameter
3399  * @hba: Pointer to adapter instance
3400  * @desc_id: descriptor idn value
3401  * @desc_index: descriptor index
3402  * @param_offset: offset of the parameter to read
3403  * @param_read_buf: pointer to buffer where parameter would be read
3404  * @param_size: sizeof(param_read_buf)
3405  *
3406  * Return 0 in case of success, non-zero otherwise
3407  */
3408 int ufshcd_read_desc_param(struct ufs_hba *hba,
3409                            enum desc_idn desc_id,
3410                            int desc_index,
3411                            u8 param_offset,
3412                            u8 *param_read_buf,
3413                            u8 param_size)
3414 {
3415         int ret;
3416         u8 *desc_buf;
3417         int buff_len;
3418         bool is_kmalloc = true;
3419
3420         /* Safety check */
3421         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3422                 return -EINVAL;
3423
3424         /* Get the length of descriptor */
3425         ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3426         if (!buff_len) {
3427                 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3428                 return -EINVAL;
3429         }
3430
3431         if (param_offset >= buff_len) {
3432                 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3433                         __func__, param_offset, desc_id, buff_len);
3434                 return -EINVAL;
3435         }
3436
3437         /* Check whether we need temp memory */
3438         if (param_offset != 0 || param_size < buff_len) {
3439                 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3440                 if (!desc_buf)
3441                         return -ENOMEM;
3442         } else {
3443                 desc_buf = param_read_buf;
3444                 is_kmalloc = false;
3445         }
3446
3447         /* Request for full descriptor */
3448         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3449                                         desc_id, desc_index, 0,
3450                                         desc_buf, &buff_len);
3451
3452         if (ret) {
3453                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3454                         __func__, desc_id, desc_index, param_offset, ret);
3455                 goto out;
3456         }
3457
3458         /* Sanity check */
3459         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3460                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3461                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3462                 ret = -EINVAL;
3463                 goto out;
3464         }
3465
3466         /* Update descriptor length */
3467         buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3468         ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3469
3470         if (is_kmalloc) {
3471                 /* Make sure we don't copy more data than available */
3472                 if (param_offset >= buff_len)
3473                         ret = -EINVAL;
3474                 else
3475                         memcpy(param_read_buf, &desc_buf[param_offset],
3476                                min_t(u32, param_size, buff_len - param_offset));
3477         }
3478 out:
3479         if (is_kmalloc)
3480                 kfree(desc_buf);
3481         return ret;
3482 }
3483
3484 /**
3485  * struct uc_string_id - unicode string
3486  *
3487  * @len: size of this descriptor inclusive
3488  * @type: descriptor type
3489  * @uc: unicode string character
3490  */
3491 struct uc_string_id {
3492         u8 len;
3493         u8 type;
3494         wchar_t uc[];
3495 } __packed;
3496
3497 /* replace non-printable or non-ASCII characters with spaces */
3498 static inline char ufshcd_remove_non_printable(u8 ch)
3499 {
3500         return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3501 }
3502
3503 /**
3504  * ufshcd_read_string_desc - read string descriptor
3505  * @hba: pointer to adapter instance
3506  * @desc_index: descriptor index
3507  * @buf: pointer to buffer where descriptor would be read,
3508  *       the caller should free the memory.
3509  * @ascii: if true convert from unicode to ascii characters
3510  *         null terminated string.
3511  *
3512  * Return:
3513  * *      string size on success.
3514  * *      -ENOMEM: on allocation failure
3515  * *      -EINVAL: on a wrong parameter
3516  */
3517 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3518                             u8 **buf, bool ascii)
3519 {
3520         struct uc_string_id *uc_str;
3521         u8 *str;
3522         int ret;
3523
3524         if (!buf)
3525                 return -EINVAL;
3526
3527         uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3528         if (!uc_str)
3529                 return -ENOMEM;
3530
3531         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3532                                      (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3533         if (ret < 0) {
3534                 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3535                         QUERY_REQ_RETRIES, ret);
3536                 str = NULL;
3537                 goto out;
3538         }
3539
3540         if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3541                 dev_dbg(hba->dev, "String Desc is of zero length\n");
3542                 str = NULL;
3543                 ret = 0;
3544                 goto out;
3545         }
3546
3547         if (ascii) {
3548                 ssize_t ascii_len;
3549                 int i;
3550                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3551                 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3552                 str = kzalloc(ascii_len, GFP_KERNEL);
3553                 if (!str) {
3554                         ret = -ENOMEM;
3555                         goto out;
3556                 }
3557
3558                 /*
3559                  * the descriptor contains string in UTF16 format
3560                  * we need to convert to utf-8 so it can be displayed
3561                  */
3562                 ret = utf16s_to_utf8s(uc_str->uc,
3563                                       uc_str->len - QUERY_DESC_HDR_SIZE,
3564                                       UTF16_BIG_ENDIAN, str, ascii_len);
3565
3566                 /* replace non-printable or non-ASCII characters with spaces */
3567                 for (i = 0; i < ret; i++)
3568                         str[i] = ufshcd_remove_non_printable(str[i]);
3569
3570                 str[ret++] = '\0';
3571
3572         } else {
3573                 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3574                 if (!str) {
3575                         ret = -ENOMEM;
3576                         goto out;
3577                 }
3578                 ret = uc_str->len;
3579         }
3580 out:
3581         *buf = str;
3582         kfree(uc_str);
3583         return ret;
3584 }
3585
3586 /**
3587  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3588  * @hba: Pointer to adapter instance
3589  * @lun: lun id
3590  * @param_offset: offset of the parameter to read
3591  * @param_read_buf: pointer to buffer where parameter would be read
3592  * @param_size: sizeof(param_read_buf)
3593  *
3594  * Return 0 in case of success, non-zero otherwise
3595  */
3596 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3597                                               int lun,
3598                                               enum unit_desc_param param_offset,
3599                                               u8 *param_read_buf,
3600                                               u32 param_size)
3601 {
3602         /*
3603          * Unit descriptors are only available for general purpose LUs (LUN id
3604          * from 0 to 7) and RPMB Well known LU.
3605          */
3606         if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
3607                 return -EOPNOTSUPP;
3608
3609         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3610                                       param_offset, param_read_buf, param_size);
3611 }
3612
3613 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3614 {
3615         int err = 0;
3616         u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3617
3618         if (hba->dev_info.wspecversion >= 0x300) {
3619                 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3620                                 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3621                                 &gating_wait);
3622                 if (err)
3623                         dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3624                                          err, gating_wait);
3625
3626                 if (gating_wait == 0) {
3627                         gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3628                         dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3629                                          gating_wait);
3630                 }
3631
3632                 hba->dev_info.clk_gating_wait_us = gating_wait;
3633         }
3634
3635         return err;
3636 }
3637
3638 /**
3639  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3640  * @hba: per adapter instance
3641  *
3642  * 1. Allocate DMA memory for Command Descriptor array
3643  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3644  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3645  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3646  *      (UTMRDL)
3647  * 4. Allocate memory for local reference block(lrb).
3648  *
3649  * Returns 0 for success, non-zero in case of failure
3650  */
3651 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3652 {
3653         size_t utmrdl_size, utrdl_size, ucdl_size;
3654
3655         /* Allocate memory for UTP command descriptors */
3656         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3657         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3658                                                   ucdl_size,
3659                                                   &hba->ucdl_dma_addr,
3660                                                   GFP_KERNEL);
3661
3662         /*
3663          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3664          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3665          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3666          * be aligned to 128 bytes as well
3667          */
3668         if (!hba->ucdl_base_addr ||
3669             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3670                 dev_err(hba->dev,
3671                         "Command Descriptor Memory allocation failed\n");
3672                 goto out;
3673         }
3674
3675         /*
3676          * Allocate memory for UTP Transfer descriptors
3677          * UFSHCI requires 1024 byte alignment of UTRD
3678          */
3679         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3680         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3681                                                    utrdl_size,
3682                                                    &hba->utrdl_dma_addr,
3683                                                    GFP_KERNEL);
3684         if (!hba->utrdl_base_addr ||
3685             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3686                 dev_err(hba->dev,
3687                         "Transfer Descriptor Memory allocation failed\n");
3688                 goto out;
3689         }
3690
3691         /*
3692          * Allocate memory for UTP Task Management descriptors
3693          * UFSHCI requires 1024 byte alignment of UTMRD
3694          */
3695         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3696         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3697                                                     utmrdl_size,
3698                                                     &hba->utmrdl_dma_addr,
3699                                                     GFP_KERNEL);
3700         if (!hba->utmrdl_base_addr ||
3701             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3702                 dev_err(hba->dev,
3703                 "Task Management Descriptor Memory allocation failed\n");
3704                 goto out;
3705         }
3706
3707         /* Allocate memory for local reference block */
3708         hba->lrb = devm_kcalloc(hba->dev,
3709                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3710                                 GFP_KERNEL);
3711         if (!hba->lrb) {
3712                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3713                 goto out;
3714         }
3715         return 0;
3716 out:
3717         return -ENOMEM;
3718 }
3719
3720 /**
3721  * ufshcd_host_memory_configure - configure local reference block with
3722  *                              memory offsets
3723  * @hba: per adapter instance
3724  *
3725  * Configure Host memory space
3726  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3727  * address.
3728  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3729  * and PRDT offset.
3730  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3731  * into local reference block.
3732  */
3733 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3734 {
3735         struct utp_transfer_req_desc *utrdlp;
3736         dma_addr_t cmd_desc_dma_addr;
3737         dma_addr_t cmd_desc_element_addr;
3738         u16 response_offset;
3739         u16 prdt_offset;
3740         int cmd_desc_size;
3741         int i;
3742
3743         utrdlp = hba->utrdl_base_addr;
3744
3745         response_offset =
3746                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3747         prdt_offset =
3748                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3749
3750         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3751         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3752
3753         for (i = 0; i < hba->nutrs; i++) {
3754                 /* Configure UTRD with command descriptor base address */
3755                 cmd_desc_element_addr =
3756                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3757                 utrdlp[i].command_desc_base_addr_lo =
3758                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3759                 utrdlp[i].command_desc_base_addr_hi =
3760                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3761
3762                 /* Response upiu and prdt offset should be in double words */
3763                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3764                         utrdlp[i].response_upiu_offset =
3765                                 cpu_to_le16(response_offset);
3766                         utrdlp[i].prd_table_offset =
3767                                 cpu_to_le16(prdt_offset);
3768                         utrdlp[i].response_upiu_length =
3769                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3770                 } else {
3771                         utrdlp[i].response_upiu_offset =
3772                                 cpu_to_le16(response_offset >> 2);
3773                         utrdlp[i].prd_table_offset =
3774                                 cpu_to_le16(prdt_offset >> 2);
3775                         utrdlp[i].response_upiu_length =
3776                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3777                 }
3778
3779                 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3780         }
3781 }
3782
3783 /**
3784  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3785  * @hba: per adapter instance
3786  *
3787  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3788  * in order to initialize the Unipro link startup procedure.
3789  * Once the Unipro links are up, the device connected to the controller
3790  * is detected.
3791  *
3792  * Returns 0 on success, non-zero value on failure
3793  */
3794 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3795 {
3796         struct uic_command uic_cmd = {0};
3797         int ret;
3798
3799         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3800
3801         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3802         if (ret)
3803                 dev_dbg(hba->dev,
3804                         "dme-link-startup: error code %d\n", ret);
3805         return ret;
3806 }
3807 /**
3808  * ufshcd_dme_reset - UIC command for DME_RESET
3809  * @hba: per adapter instance
3810  *
3811  * DME_RESET command is issued in order to reset UniPro stack.
3812  * This function now deals with cold reset.
3813  *
3814  * Returns 0 on success, non-zero value on failure
3815  */
3816 static int ufshcd_dme_reset(struct ufs_hba *hba)
3817 {
3818         struct uic_command uic_cmd = {0};
3819         int ret;
3820
3821         uic_cmd.command = UIC_CMD_DME_RESET;
3822
3823         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3824         if (ret)
3825                 dev_err(hba->dev,
3826                         "dme-reset: error code %d\n", ret);
3827
3828         return ret;
3829 }
3830
3831 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3832                                int agreed_gear,
3833                                int adapt_val)
3834 {
3835         int ret;
3836
3837         if (agreed_gear != UFS_HS_G4)
3838                 adapt_val = PA_NO_ADAPT;
3839
3840         ret = ufshcd_dme_set(hba,
3841                              UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3842                              adapt_val);
3843         return ret;
3844 }
3845 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3846
3847 /**
3848  * ufshcd_dme_enable - UIC command for DME_ENABLE
3849  * @hba: per adapter instance
3850  *
3851  * DME_ENABLE command is issued in order to enable UniPro stack.
3852  *
3853  * Returns 0 on success, non-zero value on failure
3854  */
3855 static int ufshcd_dme_enable(struct ufs_hba *hba)
3856 {
3857         struct uic_command uic_cmd = {0};
3858         int ret;
3859
3860         uic_cmd.command = UIC_CMD_DME_ENABLE;
3861
3862         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3863         if (ret)
3864                 dev_err(hba->dev,
3865                         "dme-enable: error code %d\n", ret);
3866
3867         return ret;
3868 }
3869
3870 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3871 {
3872         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3873         unsigned long min_sleep_time_us;
3874
3875         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3876                 return;
3877
3878         /*
3879          * last_dme_cmd_tstamp will be 0 only for 1st call to
3880          * this function
3881          */
3882         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3883                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3884         } else {
3885                 unsigned long delta =
3886                         (unsigned long) ktime_to_us(
3887                                 ktime_sub(ktime_get(),
3888                                 hba->last_dme_cmd_tstamp));
3889
3890                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3891                         min_sleep_time_us =
3892                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3893                 else
3894                         return; /* no more delay required */
3895         }
3896
3897         /* allow sleep for extra 50us if needed */
3898         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3899 }
3900
3901 /**
3902  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3903  * @hba: per adapter instance
3904  * @attr_sel: uic command argument1
3905  * @attr_set: attribute set type as uic command argument2
3906  * @mib_val: setting value as uic command argument3
3907  * @peer: indicate whether peer or local
3908  *
3909  * Returns 0 on success, non-zero value on failure
3910  */
3911 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3912                         u8 attr_set, u32 mib_val, u8 peer)
3913 {
3914         struct uic_command uic_cmd = {0};
3915         static const char *const action[] = {
3916                 "dme-set",
3917                 "dme-peer-set"
3918         };
3919         const char *set = action[!!peer];
3920         int ret;
3921         int retries = UFS_UIC_COMMAND_RETRIES;
3922
3923         uic_cmd.command = peer ?
3924                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3925         uic_cmd.argument1 = attr_sel;
3926         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3927         uic_cmd.argument3 = mib_val;
3928
3929         do {
3930                 /* for peer attributes we retry upon failure */
3931                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3932                 if (ret)
3933                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3934                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3935         } while (ret && peer && --retries);
3936
3937         if (ret)
3938                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3939                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3940                         UFS_UIC_COMMAND_RETRIES - retries);
3941
3942         return ret;
3943 }
3944 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3945
3946 /**
3947  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3948  * @hba: per adapter instance
3949  * @attr_sel: uic command argument1
3950  * @mib_val: the value of the attribute as returned by the UIC command
3951  * @peer: indicate whether peer or local
3952  *
3953  * Returns 0 on success, non-zero value on failure
3954  */
3955 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3956                         u32 *mib_val, u8 peer)
3957 {
3958         struct uic_command uic_cmd = {0};
3959         static const char *const action[] = {
3960                 "dme-get",
3961                 "dme-peer-get"
3962         };
3963         const char *get = action[!!peer];
3964         int ret;
3965         int retries = UFS_UIC_COMMAND_RETRIES;
3966         struct ufs_pa_layer_attr orig_pwr_info;
3967         struct ufs_pa_layer_attr temp_pwr_info;
3968         bool pwr_mode_change = false;
3969
3970         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3971                 orig_pwr_info = hba->pwr_info;
3972                 temp_pwr_info = orig_pwr_info;
3973
3974                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3975                     orig_pwr_info.pwr_rx == FAST_MODE) {
3976                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3977                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3978                         pwr_mode_change = true;
3979                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3980                     orig_pwr_info.pwr_rx == SLOW_MODE) {
3981                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3982                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3983                         pwr_mode_change = true;
3984                 }
3985                 if (pwr_mode_change) {
3986                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3987                         if (ret)
3988                                 goto out;
3989                 }
3990         }
3991
3992         uic_cmd.command = peer ?
3993                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3994         uic_cmd.argument1 = attr_sel;
3995
3996         do {
3997                 /* for peer attributes we retry upon failure */
3998                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3999                 if (ret)
4000                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4001                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
4002         } while (ret && peer && --retries);
4003
4004         if (ret)
4005                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4006                         get, UIC_GET_ATTR_ID(attr_sel),
4007                         UFS_UIC_COMMAND_RETRIES - retries);
4008
4009         if (mib_val && !ret)
4010                 *mib_val = uic_cmd.argument3;
4011
4012         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4013             && pwr_mode_change)
4014                 ufshcd_change_power_mode(hba, &orig_pwr_info);
4015 out:
4016         return ret;
4017 }
4018 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4019
4020 /**
4021  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4022  * state) and waits for it to take effect.
4023  *
4024  * @hba: per adapter instance
4025  * @cmd: UIC command to execute
4026  *
4027  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4028  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4029  * and device UniPro link and hence it's final completion would be indicated by
4030  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4031  * addition to normal UIC command completion Status (UCCS). This function only
4032  * returns after the relevant status bits indicate the completion.
4033  *
4034  * Returns 0 on success, non-zero value on failure
4035  */
4036 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4037 {
4038         DECLARE_COMPLETION_ONSTACK(uic_async_done);
4039         unsigned long flags;
4040         u8 status;
4041         int ret;
4042         bool reenable_intr = false;
4043
4044         mutex_lock(&hba->uic_cmd_mutex);
4045         ufshcd_add_delay_before_dme_cmd(hba);
4046
4047         spin_lock_irqsave(hba->host->host_lock, flags);
4048         if (ufshcd_is_link_broken(hba)) {
4049                 ret = -ENOLINK;
4050                 goto out_unlock;
4051         }
4052         hba->uic_async_done = &uic_async_done;
4053         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4054                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4055                 /*
4056                  * Make sure UIC command completion interrupt is disabled before
4057                  * issuing UIC command.
4058                  */
4059                 wmb();
4060                 reenable_intr = true;
4061         }
4062         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4063         spin_unlock_irqrestore(hba->host->host_lock, flags);
4064         if (ret) {
4065                 dev_err(hba->dev,
4066                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4067                         cmd->command, cmd->argument3, ret);
4068                 goto out;
4069         }
4070
4071         if (!wait_for_completion_timeout(hba->uic_async_done,
4072                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
4073                 dev_err(hba->dev,
4074                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4075                         cmd->command, cmd->argument3);
4076
4077                 if (!cmd->cmd_active) {
4078                         dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4079                                 __func__);
4080                         goto check_upmcrs;
4081                 }
4082
4083                 ret = -ETIMEDOUT;
4084                 goto out;
4085         }
4086
4087 check_upmcrs:
4088         status = ufshcd_get_upmcrs(hba);
4089         if (status != PWR_LOCAL) {
4090                 dev_err(hba->dev,
4091                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4092                         cmd->command, status);
4093                 ret = (status != PWR_OK) ? status : -1;
4094         }
4095 out:
4096         if (ret) {
4097                 ufshcd_print_host_state(hba);
4098                 ufshcd_print_pwr_info(hba);
4099                 ufshcd_print_evt_hist(hba);
4100         }
4101
4102         spin_lock_irqsave(hba->host->host_lock, flags);
4103         hba->active_uic_cmd = NULL;
4104         hba->uic_async_done = NULL;
4105         if (reenable_intr)
4106                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4107         if (ret) {
4108                 ufshcd_set_link_broken(hba);
4109                 ufshcd_schedule_eh_work(hba);
4110         }
4111 out_unlock:
4112         spin_unlock_irqrestore(hba->host->host_lock, flags);
4113         mutex_unlock(&hba->uic_cmd_mutex);
4114
4115         return ret;
4116 }
4117
4118 /**
4119  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4120  *                              using DME_SET primitives.
4121  * @hba: per adapter instance
4122  * @mode: powr mode value
4123  *
4124  * Returns 0 on success, non-zero value on failure
4125  */
4126 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4127 {
4128         struct uic_command uic_cmd = {0};
4129         int ret;
4130
4131         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4132                 ret = ufshcd_dme_set(hba,
4133                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4134                 if (ret) {
4135                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4136                                                 __func__, ret);
4137                         goto out;
4138                 }
4139         }
4140
4141         uic_cmd.command = UIC_CMD_DME_SET;
4142         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4143         uic_cmd.argument3 = mode;
4144         ufshcd_hold(hba, false);
4145         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4146         ufshcd_release(hba);
4147
4148 out:
4149         return ret;
4150 }
4151
4152 int ufshcd_link_recovery(struct ufs_hba *hba)
4153 {
4154         int ret;
4155         unsigned long flags;
4156
4157         spin_lock_irqsave(hba->host->host_lock, flags);
4158         hba->ufshcd_state = UFSHCD_STATE_RESET;
4159         ufshcd_set_eh_in_progress(hba);
4160         spin_unlock_irqrestore(hba->host->host_lock, flags);
4161
4162         /* Reset the attached device */
4163         ufshcd_device_reset(hba);
4164
4165         ret = ufshcd_host_reset_and_restore(hba);
4166
4167         spin_lock_irqsave(hba->host->host_lock, flags);
4168         if (ret)
4169                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4170         ufshcd_clear_eh_in_progress(hba);
4171         spin_unlock_irqrestore(hba->host->host_lock, flags);
4172
4173         if (ret)
4174                 dev_err(hba->dev, "%s: link recovery failed, err %d",
4175                         __func__, ret);
4176
4177         return ret;
4178 }
4179 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4180
4181 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4182 {
4183         int ret;
4184         struct uic_command uic_cmd = {0};
4185         ktime_t start = ktime_get();
4186
4187         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4188
4189         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4190         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4191         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4192                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4193
4194         if (ret)
4195                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4196                         __func__, ret);
4197         else
4198                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4199                                                                 POST_CHANGE);
4200
4201         return ret;
4202 }
4203 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4204
4205 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4206 {
4207         struct uic_command uic_cmd = {0};
4208         int ret;
4209         ktime_t start = ktime_get();
4210
4211         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4212
4213         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4214         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4215         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4216                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4217
4218         if (ret) {
4219                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4220                         __func__, ret);
4221         } else {
4222                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4223                                                                 POST_CHANGE);
4224                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4225                 hba->ufs_stats.hibern8_exit_cnt++;
4226         }
4227
4228         return ret;
4229 }
4230 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4231
4232 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4233 {
4234         unsigned long flags;
4235         bool update = false;
4236
4237         if (!ufshcd_is_auto_hibern8_supported(hba))
4238                 return;
4239
4240         spin_lock_irqsave(hba->host->host_lock, flags);
4241         if (hba->ahit != ahit) {
4242                 hba->ahit = ahit;
4243                 update = true;
4244         }
4245         spin_unlock_irqrestore(hba->host->host_lock, flags);
4246
4247         if (update &&
4248             !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4249                 ufshcd_rpm_get_sync(hba);
4250                 ufshcd_hold(hba, false);
4251                 ufshcd_auto_hibern8_enable(hba);
4252                 ufshcd_release(hba);
4253                 ufshcd_rpm_put_sync(hba);
4254         }
4255 }
4256 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4257
4258 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4259 {
4260         if (!ufshcd_is_auto_hibern8_supported(hba))
4261                 return;
4262
4263         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4264 }
4265
4266  /**
4267  * ufshcd_init_pwr_info - setting the POR (power on reset)
4268  * values in hba power info
4269  * @hba: per-adapter instance
4270  */
4271 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4272 {
4273         hba->pwr_info.gear_rx = UFS_PWM_G1;
4274         hba->pwr_info.gear_tx = UFS_PWM_G1;
4275         hba->pwr_info.lane_rx = 1;
4276         hba->pwr_info.lane_tx = 1;
4277         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4278         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4279         hba->pwr_info.hs_rate = 0;
4280 }
4281
4282 /**
4283  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4284  * @hba: per-adapter instance
4285  */
4286 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4287 {
4288         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4289
4290         if (hba->max_pwr_info.is_valid)
4291                 return 0;
4292
4293         pwr_info->pwr_tx = FAST_MODE;
4294         pwr_info->pwr_rx = FAST_MODE;
4295         pwr_info->hs_rate = PA_HS_MODE_B;
4296
4297         /* Get the connected lane count */
4298         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4299                         &pwr_info->lane_rx);
4300         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4301                         &pwr_info->lane_tx);
4302
4303         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4304                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4305                                 __func__,
4306                                 pwr_info->lane_rx,
4307                                 pwr_info->lane_tx);
4308                 return -EINVAL;
4309         }
4310
4311         /*
4312          * First, get the maximum gears of HS speed.
4313          * If a zero value, it means there is no HSGEAR capability.
4314          * Then, get the maximum gears of PWM speed.
4315          */
4316         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4317         if (!pwr_info->gear_rx) {
4318                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4319                                 &pwr_info->gear_rx);
4320                 if (!pwr_info->gear_rx) {
4321                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4322                                 __func__, pwr_info->gear_rx);
4323                         return -EINVAL;
4324                 }
4325                 pwr_info->pwr_rx = SLOW_MODE;
4326         }
4327
4328         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4329                         &pwr_info->gear_tx);
4330         if (!pwr_info->gear_tx) {
4331                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4332                                 &pwr_info->gear_tx);
4333                 if (!pwr_info->gear_tx) {
4334                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4335                                 __func__, pwr_info->gear_tx);
4336                         return -EINVAL;
4337                 }
4338                 pwr_info->pwr_tx = SLOW_MODE;
4339         }
4340
4341         hba->max_pwr_info.is_valid = true;
4342         return 0;
4343 }
4344
4345 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4346                              struct ufs_pa_layer_attr *pwr_mode)
4347 {
4348         int ret;
4349
4350         /* if already configured to the requested pwr_mode */
4351         if (!hba->force_pmc &&
4352             pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4353             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4354             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4355             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4356             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4357             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4358             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4359                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4360                 return 0;
4361         }
4362
4363         /*
4364          * Configure attributes for power mode change with below.
4365          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4366          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4367          * - PA_HSSERIES
4368          */
4369         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4370         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4371                         pwr_mode->lane_rx);
4372         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4373                         pwr_mode->pwr_rx == FAST_MODE)
4374                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4375         else
4376                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4377
4378         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4379         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4380                         pwr_mode->lane_tx);
4381         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4382                         pwr_mode->pwr_tx == FAST_MODE)
4383                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4384         else
4385                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4386
4387         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4388             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4389             pwr_mode->pwr_rx == FAST_MODE ||
4390             pwr_mode->pwr_tx == FAST_MODE)
4391                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4392                                                 pwr_mode->hs_rate);
4393
4394         if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4395                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4396                                 DL_FC0ProtectionTimeOutVal_Default);
4397                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4398                                 DL_TC0ReplayTimeOutVal_Default);
4399                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4400                                 DL_AFC0ReqTimeOutVal_Default);
4401                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4402                                 DL_FC1ProtectionTimeOutVal_Default);
4403                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4404                                 DL_TC1ReplayTimeOutVal_Default);
4405                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4406                                 DL_AFC1ReqTimeOutVal_Default);
4407
4408                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4409                                 DL_FC0ProtectionTimeOutVal_Default);
4410                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4411                                 DL_TC0ReplayTimeOutVal_Default);
4412                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4413                                 DL_AFC0ReqTimeOutVal_Default);
4414         }
4415
4416         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4417                         | pwr_mode->pwr_tx);
4418
4419         if (ret) {
4420                 dev_err(hba->dev,
4421                         "%s: power mode change failed %d\n", __func__, ret);
4422         } else {
4423                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4424                                                                 pwr_mode);
4425
4426                 memcpy(&hba->pwr_info, pwr_mode,
4427                         sizeof(struct ufs_pa_layer_attr));
4428         }
4429
4430         return ret;
4431 }
4432
4433 /**
4434  * ufshcd_config_pwr_mode - configure a new power mode
4435  * @hba: per-adapter instance
4436  * @desired_pwr_mode: desired power configuration
4437  */
4438 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4439                 struct ufs_pa_layer_attr *desired_pwr_mode)
4440 {
4441         struct ufs_pa_layer_attr final_params = { 0 };
4442         int ret;
4443
4444         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4445                                         desired_pwr_mode, &final_params);
4446
4447         if (ret)
4448                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4449
4450         ret = ufshcd_change_power_mode(hba, &final_params);
4451
4452         return ret;
4453 }
4454 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4455
4456 /**
4457  * ufshcd_complete_dev_init() - checks device readiness
4458  * @hba: per-adapter instance
4459  *
4460  * Set fDeviceInit flag and poll until device toggles it.
4461  */
4462 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4463 {
4464         int err;
4465         bool flag_res = true;
4466         ktime_t timeout;
4467
4468         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4469                 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4470         if (err) {
4471                 dev_err(hba->dev,
4472                         "%s setting fDeviceInit flag failed with error %d\n",
4473                         __func__, err);
4474                 goto out;
4475         }
4476
4477         /* Poll fDeviceInit flag to be cleared */
4478         timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4479         do {
4480                 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4481                                         QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4482                 if (!flag_res)
4483                         break;
4484                 usleep_range(500, 1000);
4485         } while (ktime_before(ktime_get(), timeout));
4486
4487         if (err) {
4488                 dev_err(hba->dev,
4489                                 "%s reading fDeviceInit flag failed with error %d\n",
4490                                 __func__, err);
4491         } else if (flag_res) {
4492                 dev_err(hba->dev,
4493                                 "%s fDeviceInit was not cleared by the device\n",
4494                                 __func__);
4495                 err = -EBUSY;
4496         }
4497 out:
4498         return err;
4499 }
4500
4501 /**
4502  * ufshcd_make_hba_operational - Make UFS controller operational
4503  * @hba: per adapter instance
4504  *
4505  * To bring UFS host controller to operational state,
4506  * 1. Enable required interrupts
4507  * 2. Configure interrupt aggregation
4508  * 3. Program UTRL and UTMRL base address
4509  * 4. Configure run-stop-registers
4510  *
4511  * Returns 0 on success, non-zero value on failure
4512  */
4513 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4514 {
4515         int err = 0;
4516         u32 reg;
4517
4518         /* Enable required interrupts */
4519         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4520
4521         /* Configure interrupt aggregation */
4522         if (ufshcd_is_intr_aggr_allowed(hba))
4523                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4524         else
4525                 ufshcd_disable_intr_aggr(hba);
4526
4527         /* Configure UTRL and UTMRL base address registers */
4528         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4529                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4530         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4531                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4532         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4533                         REG_UTP_TASK_REQ_LIST_BASE_L);
4534         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4535                         REG_UTP_TASK_REQ_LIST_BASE_H);
4536
4537         /*
4538          * Make sure base address and interrupt setup are updated before
4539          * enabling the run/stop registers below.
4540          */
4541         wmb();
4542
4543         /*
4544          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4545          */
4546         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4547         if (!(ufshcd_get_lists_status(reg))) {
4548                 ufshcd_enable_run_stop_reg(hba);
4549         } else {
4550                 dev_err(hba->dev,
4551                         "Host controller not ready to process requests");
4552                 err = -EIO;
4553         }
4554
4555         return err;
4556 }
4557 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4558
4559 /**
4560  * ufshcd_hba_stop - Send controller to reset state
4561  * @hba: per adapter instance
4562  */
4563 void ufshcd_hba_stop(struct ufs_hba *hba)
4564 {
4565         unsigned long flags;
4566         int err;
4567
4568         /*
4569          * Obtain the host lock to prevent that the controller is disabled
4570          * while the UFS interrupt handler is active on another CPU.
4571          */
4572         spin_lock_irqsave(hba->host->host_lock, flags);
4573         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4574         spin_unlock_irqrestore(hba->host->host_lock, flags);
4575
4576         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4577                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4578                                         10, 1);
4579         if (err)
4580                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4581 }
4582 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4583
4584 /**
4585  * ufshcd_hba_execute_hce - initialize the controller
4586  * @hba: per adapter instance
4587  *
4588  * The controller resets itself and controller firmware initialization
4589  * sequence kicks off. When controller is ready it will set
4590  * the Host Controller Enable bit to 1.
4591  *
4592  * Returns 0 on success, non-zero value on failure
4593  */
4594 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4595 {
4596         int retry_outer = 3;
4597         int retry_inner;
4598
4599 start:
4600         if (ufshcd_is_hba_active(hba))
4601                 /* change controller state to "reset state" */
4602                 ufshcd_hba_stop(hba);
4603
4604         /* UniPro link is disabled at this point */
4605         ufshcd_set_link_off(hba);
4606
4607         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4608
4609         /* start controller initialization sequence */
4610         ufshcd_hba_start(hba);
4611
4612         /*
4613          * To initialize a UFS host controller HCE bit must be set to 1.
4614          * During initialization the HCE bit value changes from 1->0->1.
4615          * When the host controller completes initialization sequence
4616          * it sets the value of HCE bit to 1. The same HCE bit is read back
4617          * to check if the controller has completed initialization sequence.
4618          * So without this delay the value HCE = 1, set in the previous
4619          * instruction might be read back.
4620          * This delay can be changed based on the controller.
4621          */
4622         ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4623
4624         /* wait for the host controller to complete initialization */
4625         retry_inner = 50;
4626         while (!ufshcd_is_hba_active(hba)) {
4627                 if (retry_inner) {
4628                         retry_inner--;
4629                 } else {
4630                         dev_err(hba->dev,
4631                                 "Controller enable failed\n");
4632                         if (retry_outer) {
4633                                 retry_outer--;
4634                                 goto start;
4635                         }
4636                         return -EIO;
4637                 }
4638                 usleep_range(1000, 1100);
4639         }
4640
4641         /* enable UIC related interrupts */
4642         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4643
4644         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4645
4646         return 0;
4647 }
4648
4649 int ufshcd_hba_enable(struct ufs_hba *hba)
4650 {
4651         int ret;
4652
4653         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4654                 ufshcd_set_link_off(hba);
4655                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4656
4657                 /* enable UIC related interrupts */
4658                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4659                 ret = ufshcd_dme_reset(hba);
4660                 if (!ret) {
4661                         ret = ufshcd_dme_enable(hba);
4662                         if (!ret)
4663                                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4664                         if (ret)
4665                                 dev_err(hba->dev,
4666                                         "Host controller enable failed with non-hce\n");
4667                 }
4668         } else {
4669                 ret = ufshcd_hba_execute_hce(hba);
4670         }
4671
4672         return ret;
4673 }
4674 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4675
4676 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4677 {
4678         int tx_lanes = 0, i, err = 0;
4679
4680         if (!peer)
4681                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4682                                &tx_lanes);
4683         else
4684                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4685                                     &tx_lanes);
4686         for (i = 0; i < tx_lanes; i++) {
4687                 if (!peer)
4688                         err = ufshcd_dme_set(hba,
4689                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4690                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4691                                         0);
4692                 else
4693                         err = ufshcd_dme_peer_set(hba,
4694                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4695                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4696                                         0);
4697                 if (err) {
4698                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4699                                 __func__, peer, i, err);
4700                         break;
4701                 }
4702         }
4703
4704         return err;
4705 }
4706
4707 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4708 {
4709         return ufshcd_disable_tx_lcc(hba, true);
4710 }
4711
4712 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4713 {
4714         struct ufs_event_hist *e;
4715
4716         if (id >= UFS_EVT_CNT)
4717                 return;
4718
4719         e = &hba->ufs_stats.event[id];
4720         e->val[e->pos] = val;
4721         e->tstamp[e->pos] = ktime_get();
4722         e->cnt += 1;
4723         e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4724
4725         ufshcd_vops_event_notify(hba, id, &val);
4726 }
4727 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4728
4729 /**
4730  * ufshcd_link_startup - Initialize unipro link startup
4731  * @hba: per adapter instance
4732  *
4733  * Returns 0 for success, non-zero in case of failure
4734  */
4735 static int ufshcd_link_startup(struct ufs_hba *hba)
4736 {
4737         int ret;
4738         int retries = DME_LINKSTARTUP_RETRIES;
4739         bool link_startup_again = false;
4740
4741         /*
4742          * If UFS device isn't active then we will have to issue link startup
4743          * 2 times to make sure the device state move to active.
4744          */
4745         if (!ufshcd_is_ufs_dev_active(hba))
4746                 link_startup_again = true;
4747
4748 link_startup:
4749         do {
4750                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4751
4752                 ret = ufshcd_dme_link_startup(hba);
4753
4754                 /* check if device is detected by inter-connect layer */
4755                 if (!ret && !ufshcd_is_device_present(hba)) {
4756                         ufshcd_update_evt_hist(hba,
4757                                                UFS_EVT_LINK_STARTUP_FAIL,
4758                                                0);
4759                         dev_err(hba->dev, "%s: Device not present\n", __func__);
4760                         ret = -ENXIO;
4761                         goto out;
4762                 }
4763
4764                 /*
4765                  * DME link lost indication is only received when link is up,
4766                  * but we can't be sure if the link is up until link startup
4767                  * succeeds. So reset the local Uni-Pro and try again.
4768                  */
4769                 if (ret && ufshcd_hba_enable(hba)) {
4770                         ufshcd_update_evt_hist(hba,
4771                                                UFS_EVT_LINK_STARTUP_FAIL,
4772                                                (u32)ret);
4773                         goto out;
4774                 }
4775         } while (ret && retries--);
4776
4777         if (ret) {
4778                 /* failed to get the link up... retire */
4779                 ufshcd_update_evt_hist(hba,
4780                                        UFS_EVT_LINK_STARTUP_FAIL,
4781                                        (u32)ret);
4782                 goto out;
4783         }
4784
4785         if (link_startup_again) {
4786                 link_startup_again = false;
4787                 retries = DME_LINKSTARTUP_RETRIES;
4788                 goto link_startup;
4789         }
4790
4791         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4792         ufshcd_init_pwr_info(hba);
4793         ufshcd_print_pwr_info(hba);
4794
4795         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4796                 ret = ufshcd_disable_device_tx_lcc(hba);
4797                 if (ret)
4798                         goto out;
4799         }
4800
4801         /* Include any host controller configuration via UIC commands */
4802         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4803         if (ret)
4804                 goto out;
4805
4806         /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4807         ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4808         ret = ufshcd_make_hba_operational(hba);
4809 out:
4810         if (ret) {
4811                 dev_err(hba->dev, "link startup failed %d\n", ret);
4812                 ufshcd_print_host_state(hba);
4813                 ufshcd_print_pwr_info(hba);
4814                 ufshcd_print_evt_hist(hba);
4815         }
4816         return ret;
4817 }
4818
4819 /**
4820  * ufshcd_verify_dev_init() - Verify device initialization
4821  * @hba: per-adapter instance
4822  *
4823  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4824  * device Transport Protocol (UTP) layer is ready after a reset.
4825  * If the UTP layer at the device side is not initialized, it may
4826  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4827  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4828  */
4829 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4830 {
4831         int err = 0;
4832         int retries;
4833
4834         ufshcd_hold(hba, false);
4835         mutex_lock(&hba->dev_cmd.lock);
4836         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4837                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4838                                           hba->nop_out_timeout);
4839
4840                 if (!err || err == -ETIMEDOUT)
4841                         break;
4842
4843                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4844         }
4845         mutex_unlock(&hba->dev_cmd.lock);
4846         ufshcd_release(hba);
4847
4848         if (err)
4849                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4850         return err;
4851 }
4852
4853 /**
4854  * ufshcd_set_queue_depth - set lun queue depth
4855  * @sdev: pointer to SCSI device
4856  *
4857  * Read bLUQueueDepth value and activate scsi tagged command
4858  * queueing. For WLUN, queue depth is set to 1. For best-effort
4859  * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4860  * value that host can queue.
4861  */
4862 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4863 {
4864         int ret = 0;
4865         u8 lun_qdepth;
4866         struct ufs_hba *hba;
4867
4868         hba = shost_priv(sdev->host);
4869
4870         lun_qdepth = hba->nutrs;
4871         ret = ufshcd_read_unit_desc_param(hba,
4872                                           ufshcd_scsi_to_upiu_lun(sdev->lun),
4873                                           UNIT_DESC_PARAM_LU_Q_DEPTH,
4874                                           &lun_qdepth,
4875                                           sizeof(lun_qdepth));
4876
4877         /* Some WLUN doesn't support unit descriptor */
4878         if (ret == -EOPNOTSUPP)
4879                 lun_qdepth = 1;
4880         else if (!lun_qdepth)
4881                 /* eventually, we can figure out the real queue depth */
4882                 lun_qdepth = hba->nutrs;
4883         else
4884                 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4885
4886         dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4887                         __func__, lun_qdepth);
4888         scsi_change_queue_depth(sdev, lun_qdepth);
4889 }
4890
4891 /*
4892  * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4893  * @hba: per-adapter instance
4894  * @lun: UFS device lun id
4895  * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4896  *
4897  * Returns 0 in case of success and b_lu_write_protect status would be returned
4898  * @b_lu_write_protect parameter.
4899  * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4900  * Returns -EINVAL in case of invalid parameters passed to this function.
4901  */
4902 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4903                             u8 lun,
4904                             u8 *b_lu_write_protect)
4905 {
4906         int ret;
4907
4908         if (!b_lu_write_protect)
4909                 ret = -EINVAL;
4910         /*
4911          * According to UFS device spec, RPMB LU can't be write
4912          * protected so skip reading bLUWriteProtect parameter for
4913          * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4914          */
4915         else if (lun >= hba->dev_info.max_lu_supported)
4916                 ret = -ENOTSUPP;
4917         else
4918                 ret = ufshcd_read_unit_desc_param(hba,
4919                                           lun,
4920                                           UNIT_DESC_PARAM_LU_WR_PROTECT,
4921                                           b_lu_write_protect,
4922                                           sizeof(*b_lu_write_protect));
4923         return ret;
4924 }
4925
4926 /**
4927  * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4928  * status
4929  * @hba: per-adapter instance
4930  * @sdev: pointer to SCSI device
4931  *
4932  */
4933 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4934                                                     struct scsi_device *sdev)
4935 {
4936         if (hba->dev_info.f_power_on_wp_en &&
4937             !hba->dev_info.is_lu_power_on_wp) {
4938                 u8 b_lu_write_protect;
4939
4940                 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4941                                       &b_lu_write_protect) &&
4942                     (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4943                         hba->dev_info.is_lu_power_on_wp = true;
4944         }
4945 }
4946
4947 /**
4948  * ufshcd_setup_links - associate link b/w device wlun and other luns
4949  * @sdev: pointer to SCSI device
4950  * @hba: pointer to ufs hba
4951  */
4952 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
4953 {
4954         struct device_link *link;
4955
4956         /*
4957          * Device wlun is the supplier & rest of the luns are consumers.
4958          * This ensures that device wlun suspends after all other luns.
4959          */
4960         if (hba->ufs_device_wlun) {
4961                 link = device_link_add(&sdev->sdev_gendev,
4962                                        &hba->ufs_device_wlun->sdev_gendev,
4963                                        DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
4964                 if (!link) {
4965                         dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
4966                                 dev_name(&hba->ufs_device_wlun->sdev_gendev));
4967                         return;
4968                 }
4969                 hba->luns_avail--;
4970                 /* Ignore REPORT_LUN wlun probing */
4971                 if (hba->luns_avail == 1) {
4972                         ufshcd_rpm_put(hba);
4973                         return;
4974                 }
4975         } else {
4976                 /*
4977                  * Device wlun is probed. The assumption is that WLUNs are
4978                  * scanned before other LUNs.
4979                  */
4980                 hba->luns_avail--;
4981         }
4982 }
4983
4984 /**
4985  * ufshcd_slave_alloc - handle initial SCSI device configurations
4986  * @sdev: pointer to SCSI device
4987  *
4988  * Returns success
4989  */
4990 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4991 {
4992         struct ufs_hba *hba;
4993
4994         hba = shost_priv(sdev->host);
4995
4996         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4997         sdev->use_10_for_ms = 1;
4998
4999         /* DBD field should be set to 1 in mode sense(10) */
5000         sdev->set_dbd_for_ms = 1;
5001
5002         /* allow SCSI layer to restart the device in case of errors */
5003         sdev->allow_restart = 1;
5004
5005         /* REPORT SUPPORTED OPERATION CODES is not supported */
5006         sdev->no_report_opcodes = 1;
5007
5008         /* WRITE_SAME command is not supported */
5009         sdev->no_write_same = 1;
5010
5011         ufshcd_set_queue_depth(sdev);
5012
5013         ufshcd_get_lu_power_on_wp_status(hba, sdev);
5014
5015         ufshcd_setup_links(hba, sdev);
5016
5017         return 0;
5018 }
5019
5020 /**
5021  * ufshcd_change_queue_depth - change queue depth
5022  * @sdev: pointer to SCSI device
5023  * @depth: required depth to set
5024  *
5025  * Change queue depth and make sure the max. limits are not crossed.
5026  */
5027 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5028 {
5029         return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5030 }
5031
5032 static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev)
5033 {
5034         /* skip well-known LU */
5035         if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5036             !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
5037                 return;
5038
5039         ufshpb_destroy_lu(hba, sdev);
5040 }
5041
5042 static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev)
5043 {
5044         /* skip well-known LU */
5045         if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5046             !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
5047                 return;
5048
5049         ufshpb_init_hpb_lu(hba, sdev);
5050 }
5051
5052 /**
5053  * ufshcd_slave_configure - adjust SCSI device configurations
5054  * @sdev: pointer to SCSI device
5055  */
5056 static int ufshcd_slave_configure(struct scsi_device *sdev)
5057 {
5058         struct ufs_hba *hba = shost_priv(sdev->host);
5059         struct request_queue *q = sdev->request_queue;
5060
5061         ufshcd_hpb_configure(hba, sdev);
5062
5063         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
5064         if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
5065                 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
5066         /*
5067          * Block runtime-pm until all consumers are added.
5068          * Refer ufshcd_setup_links().
5069          */
5070         if (is_device_wlun(sdev))
5071                 pm_runtime_get_noresume(&sdev->sdev_gendev);
5072         else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5073                 sdev->rpm_autosuspend = 1;
5074         /*
5075          * Do not print messages during runtime PM to avoid never-ending cycles
5076          * of messages written back to storage by user space causing runtime
5077          * resume, causing more messages and so on.
5078          */
5079         sdev->silence_suspend = 1;
5080
5081         ufshcd_crypto_register(hba, q);
5082
5083         return 0;
5084 }
5085
5086 /**
5087  * ufshcd_slave_destroy - remove SCSI device configurations
5088  * @sdev: pointer to SCSI device
5089  */
5090 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5091 {
5092         struct ufs_hba *hba;
5093         unsigned long flags;
5094
5095         hba = shost_priv(sdev->host);
5096
5097         ufshcd_hpb_destroy(hba, sdev);
5098
5099         /* Drop the reference as it won't be needed anymore */
5100         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5101                 spin_lock_irqsave(hba->host->host_lock, flags);
5102                 hba->ufs_device_wlun = NULL;
5103                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5104         } else if (hba->ufs_device_wlun) {
5105                 struct device *supplier = NULL;
5106
5107                 /* Ensure UFS Device WLUN exists and does not disappear */
5108                 spin_lock_irqsave(hba->host->host_lock, flags);
5109                 if (hba->ufs_device_wlun) {
5110                         supplier = &hba->ufs_device_wlun->sdev_gendev;
5111                         get_device(supplier);
5112                 }
5113                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5114
5115                 if (supplier) {
5116                         /*
5117                          * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5118                          * device will not have been registered but can still
5119                          * have a device link holding a reference to the device.
5120                          */
5121                         device_link_remove(&sdev->sdev_gendev, supplier);
5122                         put_device(supplier);
5123                 }
5124         }
5125 }
5126
5127 /**
5128  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5129  * @lrbp: pointer to local reference block of completed command
5130  * @scsi_status: SCSI command status
5131  *
5132  * Returns value base on SCSI command status
5133  */
5134 static inline int
5135 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5136 {
5137         int result = 0;
5138
5139         switch (scsi_status) {
5140         case SAM_STAT_CHECK_CONDITION:
5141                 ufshcd_copy_sense_data(lrbp);
5142                 fallthrough;
5143         case SAM_STAT_GOOD:
5144                 result |= DID_OK << 16 | scsi_status;
5145                 break;
5146         case SAM_STAT_TASK_SET_FULL:
5147         case SAM_STAT_BUSY:
5148         case SAM_STAT_TASK_ABORTED:
5149                 ufshcd_copy_sense_data(lrbp);
5150                 result |= scsi_status;
5151                 break;
5152         default:
5153                 result |= DID_ERROR << 16;
5154                 break;
5155         } /* end of switch */
5156
5157         return result;
5158 }
5159
5160 /**
5161  * ufshcd_transfer_rsp_status - Get overall status of the response
5162  * @hba: per adapter instance
5163  * @lrbp: pointer to local reference block of completed command
5164  *
5165  * Returns result of the command to notify SCSI midlayer
5166  */
5167 static inline int
5168 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5169 {
5170         int result = 0;
5171         int scsi_status;
5172         enum utp_ocs ocs;
5173
5174         /* overall command status of utrd */
5175         ocs = ufshcd_get_tr_ocs(lrbp);
5176
5177         if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5178                 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5179                                         MASK_RSP_UPIU_RESULT)
5180                         ocs = OCS_SUCCESS;
5181         }
5182
5183         switch (ocs) {
5184         case OCS_SUCCESS:
5185                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
5186                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5187                 switch (result) {
5188                 case UPIU_TRANSACTION_RESPONSE:
5189                         /*
5190                          * get the response UPIU result to extract
5191                          * the SCSI command status
5192                          */
5193                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5194
5195                         /*
5196                          * get the result based on SCSI status response
5197                          * to notify the SCSI midlayer of the command status
5198                          */
5199                         scsi_status = result & MASK_SCSI_STATUS;
5200                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5201
5202                         /*
5203                          * Currently we are only supporting BKOPs exception
5204                          * events hence we can ignore BKOPs exception event
5205                          * during power management callbacks. BKOPs exception
5206                          * event is not expected to be raised in runtime suspend
5207                          * callback as it allows the urgent bkops.
5208                          * During system suspend, we are anyway forcefully
5209                          * disabling the bkops and if urgent bkops is needed
5210                          * it will be enabled on system resume. Long term
5211                          * solution could be to abort the system suspend if
5212                          * UFS device needs urgent BKOPs.
5213                          */
5214                         if (!hba->pm_op_in_progress &&
5215                             !ufshcd_eh_in_progress(hba) &&
5216                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5217                                 /* Flushed in suspend */
5218                                 schedule_work(&hba->eeh_work);
5219
5220                         if (scsi_status == SAM_STAT_GOOD)
5221                                 ufshpb_rsp_upiu(hba, lrbp);
5222                         break;
5223                 case UPIU_TRANSACTION_REJECT_UPIU:
5224                         /* TODO: handle Reject UPIU Response */
5225                         result = DID_ERROR << 16;
5226                         dev_err(hba->dev,
5227                                 "Reject UPIU not fully implemented\n");
5228                         break;
5229                 default:
5230                         dev_err(hba->dev,
5231                                 "Unexpected request response code = %x\n",
5232                                 result);
5233                         result = DID_ERROR << 16;
5234                         break;
5235                 }
5236                 break;
5237         case OCS_ABORTED:
5238                 result |= DID_ABORT << 16;
5239                 break;
5240         case OCS_INVALID_COMMAND_STATUS:
5241                 result |= DID_REQUEUE << 16;
5242                 break;
5243         case OCS_INVALID_CMD_TABLE_ATTR:
5244         case OCS_INVALID_PRDT_ATTR:
5245         case OCS_MISMATCH_DATA_BUF_SIZE:
5246         case OCS_MISMATCH_RESP_UPIU_SIZE:
5247         case OCS_PEER_COMM_FAILURE:
5248         case OCS_FATAL_ERROR:
5249         case OCS_DEVICE_FATAL_ERROR:
5250         case OCS_INVALID_CRYPTO_CONFIG:
5251         case OCS_GENERAL_CRYPTO_ERROR:
5252         default:
5253                 result |= DID_ERROR << 16;
5254                 dev_err(hba->dev,
5255                                 "OCS error from controller = %x for tag %d\n",
5256                                 ocs, lrbp->task_tag);
5257                 ufshcd_print_evt_hist(hba);
5258                 ufshcd_print_host_state(hba);
5259                 break;
5260         } /* end of switch */
5261
5262         if ((host_byte(result) != DID_OK) &&
5263             (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5264                 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
5265         return result;
5266 }
5267
5268 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5269                                          u32 intr_mask)
5270 {
5271         if (!ufshcd_is_auto_hibern8_supported(hba) ||
5272             !ufshcd_is_auto_hibern8_enabled(hba))
5273                 return false;
5274
5275         if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5276                 return false;
5277
5278         if (hba->active_uic_cmd &&
5279             (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5280             hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5281                 return false;
5282
5283         return true;
5284 }
5285
5286 /**
5287  * ufshcd_uic_cmd_compl - handle completion of uic command
5288  * @hba: per adapter instance
5289  * @intr_status: interrupt status generated by the controller
5290  *
5291  * Returns
5292  *  IRQ_HANDLED - If interrupt is valid
5293  *  IRQ_NONE    - If invalid interrupt
5294  */
5295 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5296 {
5297         irqreturn_t retval = IRQ_NONE;
5298
5299         spin_lock(hba->host->host_lock);
5300         if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5301                 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5302
5303         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5304                 hba->active_uic_cmd->argument2 |=
5305                         ufshcd_get_uic_cmd_result(hba);
5306                 hba->active_uic_cmd->argument3 =
5307                         ufshcd_get_dme_attr_val(hba);
5308                 if (!hba->uic_async_done)
5309                         hba->active_uic_cmd->cmd_active = 0;
5310                 complete(&hba->active_uic_cmd->done);
5311                 retval = IRQ_HANDLED;
5312         }
5313
5314         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5315                 hba->active_uic_cmd->cmd_active = 0;
5316                 complete(hba->uic_async_done);
5317                 retval = IRQ_HANDLED;
5318         }
5319
5320         if (retval == IRQ_HANDLED)
5321                 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5322                                              UFS_CMD_COMP);
5323         spin_unlock(hba->host->host_lock);
5324         return retval;
5325 }
5326
5327 /* Release the resources allocated for processing a SCSI command. */
5328 static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5329                                     struct ufshcd_lrb *lrbp)
5330 {
5331         struct scsi_cmnd *cmd = lrbp->cmd;
5332
5333         scsi_dma_unmap(cmd);
5334         lrbp->cmd = NULL;       /* Mark the command as completed. */
5335         ufshcd_release(hba);
5336         ufshcd_clk_scaling_update_busy(hba);
5337 }
5338
5339 /**
5340  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5341  * @hba: per adapter instance
5342  * @completed_reqs: bitmask that indicates which requests to complete
5343  */
5344 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5345                                         unsigned long completed_reqs)
5346 {
5347         struct ufshcd_lrb *lrbp;
5348         struct scsi_cmnd *cmd;
5349         int index;
5350
5351         for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5352                 lrbp = &hba->lrb[index];
5353                 lrbp->compl_time_stamp = ktime_get();
5354                 cmd = lrbp->cmd;
5355                 if (cmd) {
5356                         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5357                                 ufshcd_update_monitor(hba, lrbp);
5358                         ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
5359                         cmd->result = ufshcd_transfer_rsp_status(hba, lrbp);
5360                         ufshcd_release_scsi_cmd(hba, lrbp);
5361                         /* Do not touch lrbp after scsi done */
5362                         scsi_done(cmd);
5363                 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5364                         lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5365                         if (hba->dev_cmd.complete) {
5366                                 ufshcd_add_command_trace(hba, index,
5367                                                          UFS_DEV_COMP);
5368                                 complete(hba->dev_cmd.complete);
5369                                 ufshcd_clk_scaling_update_busy(hba);
5370                         }
5371                 }
5372         }
5373 }
5374
5375 /*
5376  * Returns > 0 if one or more commands have been completed or 0 if no
5377  * requests have been completed.
5378  */
5379 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5380 {
5381         struct ufs_hba *hba = shost_priv(shost);
5382         unsigned long completed_reqs, flags;
5383         u32 tr_doorbell;
5384
5385         spin_lock_irqsave(&hba->outstanding_lock, flags);
5386         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5387         completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5388         WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5389                   "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5390                   hba->outstanding_reqs);
5391         hba->outstanding_reqs &= ~completed_reqs;
5392         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5393
5394         if (completed_reqs)
5395                 __ufshcd_transfer_req_compl(hba, completed_reqs);
5396
5397         return completed_reqs;
5398 }
5399
5400 /**
5401  * ufshcd_transfer_req_compl - handle SCSI and query command completion
5402  * @hba: per adapter instance
5403  *
5404  * Returns
5405  *  IRQ_HANDLED - If interrupt is valid
5406  *  IRQ_NONE    - If invalid interrupt
5407  */
5408 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5409 {
5410         /* Resetting interrupt aggregation counters first and reading the
5411          * DOOR_BELL afterward allows us to handle all the completed requests.
5412          * In order to prevent other interrupts starvation the DB is read once
5413          * after reset. The down side of this solution is the possibility of
5414          * false interrupt if device completes another request after resetting
5415          * aggregation and before reading the DB.
5416          */
5417         if (ufshcd_is_intr_aggr_allowed(hba) &&
5418             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5419                 ufshcd_reset_intr_aggr(hba);
5420
5421         if (ufs_fail_completion())
5422                 return IRQ_HANDLED;
5423
5424         /*
5425          * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5426          * do not want polling to trigger spurious interrupt complaints.
5427          */
5428         ufshcd_poll(hba->host, 0);
5429
5430         return IRQ_HANDLED;
5431 }
5432
5433 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5434 {
5435         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5436                                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5437                                        &ee_ctrl_mask);
5438 }
5439
5440 int ufshcd_write_ee_control(struct ufs_hba *hba)
5441 {
5442         int err;
5443
5444         mutex_lock(&hba->ee_ctrl_mutex);
5445         err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5446         mutex_unlock(&hba->ee_ctrl_mutex);
5447         if (err)
5448                 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5449                         __func__, err);
5450         return err;
5451 }
5452
5453 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, u16 *other_mask,
5454                              u16 set, u16 clr)
5455 {
5456         u16 new_mask, ee_ctrl_mask;
5457         int err = 0;
5458
5459         mutex_lock(&hba->ee_ctrl_mutex);
5460         new_mask = (*mask & ~clr) | set;
5461         ee_ctrl_mask = new_mask | *other_mask;
5462         if (ee_ctrl_mask != hba->ee_ctrl_mask)
5463                 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5464         /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5465         if (!err) {
5466                 hba->ee_ctrl_mask = ee_ctrl_mask;
5467                 *mask = new_mask;
5468         }
5469         mutex_unlock(&hba->ee_ctrl_mutex);
5470         return err;
5471 }
5472
5473 /**
5474  * ufshcd_disable_ee - disable exception event
5475  * @hba: per-adapter instance
5476  * @mask: exception event to disable
5477  *
5478  * Disables exception event in the device so that the EVENT_ALERT
5479  * bit is not set.
5480  *
5481  * Returns zero on success, non-zero error value on failure.
5482  */
5483 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5484 {
5485         return ufshcd_update_ee_drv_mask(hba, 0, mask);
5486 }
5487
5488 /**
5489  * ufshcd_enable_ee - enable exception event
5490  * @hba: per-adapter instance
5491  * @mask: exception event to enable
5492  *
5493  * Enable corresponding exception event in the device to allow
5494  * device to alert host in critical scenarios.
5495  *
5496  * Returns zero on success, non-zero error value on failure.
5497  */
5498 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5499 {
5500         return ufshcd_update_ee_drv_mask(hba, mask, 0);
5501 }
5502
5503 /**
5504  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5505  * @hba: per-adapter instance
5506  *
5507  * Allow device to manage background operations on its own. Enabling
5508  * this might lead to inconsistent latencies during normal data transfers
5509  * as the device is allowed to manage its own way of handling background
5510  * operations.
5511  *
5512  * Returns zero on success, non-zero on failure.
5513  */
5514 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5515 {
5516         int err = 0;
5517
5518         if (hba->auto_bkops_enabled)
5519                 goto out;
5520
5521         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5522                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5523         if (err) {
5524                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5525                                 __func__, err);
5526                 goto out;
5527         }
5528
5529         hba->auto_bkops_enabled = true;
5530         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5531
5532         /* No need of URGENT_BKOPS exception from the device */
5533         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5534         if (err)
5535                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5536                                 __func__, err);
5537 out:
5538         return err;
5539 }
5540
5541 /**
5542  * ufshcd_disable_auto_bkops - block device in doing background operations
5543  * @hba: per-adapter instance
5544  *
5545  * Disabling background operations improves command response latency but
5546  * has drawback of device moving into critical state where the device is
5547  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5548  * host is idle so that BKOPS are managed effectively without any negative
5549  * impacts.
5550  *
5551  * Returns zero on success, non-zero on failure.
5552  */
5553 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5554 {
5555         int err = 0;
5556
5557         if (!hba->auto_bkops_enabled)
5558                 goto out;
5559
5560         /*
5561          * If host assisted BKOPs is to be enabled, make sure
5562          * urgent bkops exception is allowed.
5563          */
5564         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5565         if (err) {
5566                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5567                                 __func__, err);
5568                 goto out;
5569         }
5570
5571         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5572                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5573         if (err) {
5574                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5575                                 __func__, err);
5576                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5577                 goto out;
5578         }
5579
5580         hba->auto_bkops_enabled = false;
5581         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5582         hba->is_urgent_bkops_lvl_checked = false;
5583 out:
5584         return err;
5585 }
5586
5587 /**
5588  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5589  * @hba: per adapter instance
5590  *
5591  * After a device reset the device may toggle the BKOPS_EN flag
5592  * to default value. The s/w tracking variables should be updated
5593  * as well. This function would change the auto-bkops state based on
5594  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5595  */
5596 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5597 {
5598         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5599                 hba->auto_bkops_enabled = false;
5600                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5601                 ufshcd_enable_auto_bkops(hba);
5602         } else {
5603                 hba->auto_bkops_enabled = true;
5604                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5605                 ufshcd_disable_auto_bkops(hba);
5606         }
5607         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5608         hba->is_urgent_bkops_lvl_checked = false;
5609 }
5610
5611 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5612 {
5613         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5614                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5615 }
5616
5617 /**
5618  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5619  * @hba: per-adapter instance
5620  * @status: bkops_status value
5621  *
5622  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5623  * flag in the device to permit background operations if the device
5624  * bkops_status is greater than or equal to "status" argument passed to
5625  * this function, disable otherwise.
5626  *
5627  * Returns 0 for success, non-zero in case of failure.
5628  *
5629  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5630  * to know whether auto bkops is enabled or disabled after this function
5631  * returns control to it.
5632  */
5633 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5634                              enum bkops_status status)
5635 {
5636         int err;
5637         u32 curr_status = 0;
5638
5639         err = ufshcd_get_bkops_status(hba, &curr_status);
5640         if (err) {
5641                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5642                                 __func__, err);
5643                 goto out;
5644         } else if (curr_status > BKOPS_STATUS_MAX) {
5645                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5646                                 __func__, curr_status);
5647                 err = -EINVAL;
5648                 goto out;
5649         }
5650
5651         if (curr_status >= status)
5652                 err = ufshcd_enable_auto_bkops(hba);
5653         else
5654                 err = ufshcd_disable_auto_bkops(hba);
5655 out:
5656         return err;
5657 }
5658
5659 /**
5660  * ufshcd_urgent_bkops - handle urgent bkops exception event
5661  * @hba: per-adapter instance
5662  *
5663  * Enable fBackgroundOpsEn flag in the device to permit background
5664  * operations.
5665  *
5666  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5667  * and negative error value for any other failure.
5668  */
5669 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5670 {
5671         return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5672 }
5673
5674 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5675 {
5676         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5677                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5678 }
5679
5680 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5681 {
5682         int err;
5683         u32 curr_status = 0;
5684
5685         if (hba->is_urgent_bkops_lvl_checked)
5686                 goto enable_auto_bkops;
5687
5688         err = ufshcd_get_bkops_status(hba, &curr_status);
5689         if (err) {
5690                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5691                                 __func__, err);
5692                 goto out;
5693         }
5694
5695         /*
5696          * We are seeing that some devices are raising the urgent bkops
5697          * exception events even when BKOPS status doesn't indicate performace
5698          * impacted or critical. Handle these device by determining their urgent
5699          * bkops status at runtime.
5700          */
5701         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5702                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5703                                 __func__, curr_status);
5704                 /* update the current status as the urgent bkops level */
5705                 hba->urgent_bkops_lvl = curr_status;
5706                 hba->is_urgent_bkops_lvl_checked = true;
5707         }
5708
5709 enable_auto_bkops:
5710         err = ufshcd_enable_auto_bkops(hba);
5711 out:
5712         if (err < 0)
5713                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5714                                 __func__, err);
5715 }
5716
5717 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5718 {
5719         u32 value;
5720
5721         if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5722                                 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5723                 return;
5724
5725         dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5726
5727         ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5728
5729         /*
5730          * A placeholder for the platform vendors to add whatever additional
5731          * steps required
5732          */
5733 }
5734
5735 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5736 {
5737         u8 index;
5738         enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5739                                    UPIU_QUERY_OPCODE_CLEAR_FLAG;
5740
5741         index = ufshcd_wb_get_query_index(hba);
5742         return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
5743 }
5744
5745 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
5746 {
5747         int ret;
5748
5749         if (!ufshcd_is_wb_allowed(hba))
5750                 return 0;
5751
5752         if (!(enable ^ hba->dev_info.wb_enabled))
5753                 return 0;
5754
5755         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
5756         if (ret) {
5757                 dev_err(hba->dev, "%s Write Booster %s failed %d\n",
5758                         __func__, enable ? "enable" : "disable", ret);
5759                 return ret;
5760         }
5761
5762         hba->dev_info.wb_enabled = enable;
5763         dev_dbg(hba->dev, "%s Write Booster %s\n",
5764                         __func__, enable ? "enabled" : "disabled");
5765
5766         return ret;
5767 }
5768
5769 static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5770 {
5771         int ret;
5772
5773         ret = __ufshcd_wb_toggle(hba, set,
5774                         QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
5775         if (ret) {
5776                 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed: %d\n",
5777                         __func__, set ? "enable" : "disable", ret);
5778                 return;
5779         }
5780         dev_dbg(hba->dev, "%s WB-Buf Flush during H8 %s\n",
5781                         __func__, set ? "enabled" : "disabled");
5782 }
5783
5784 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5785 {
5786         int ret;
5787
5788         if (!ufshcd_is_wb_allowed(hba) ||
5789             hba->dev_info.wb_buf_flush_enabled == enable)
5790                 return;
5791
5792         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
5793         if (ret) {
5794                 dev_err(hba->dev, "%s WB-Buf Flush %s failed %d\n", __func__,
5795                         enable ? "enable" : "disable", ret);
5796                 return;
5797         }
5798
5799         hba->dev_info.wb_buf_flush_enabled = enable;
5800
5801         dev_dbg(hba->dev, "%s WB-Buf Flush %s\n",
5802                         __func__, enable ? "enabled" : "disabled");
5803 }
5804
5805 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5806                                                 u32 avail_buf)
5807 {
5808         u32 cur_buf;
5809         int ret;
5810         u8 index;
5811
5812         index = ufshcd_wb_get_query_index(hba);
5813         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5814                                               QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
5815                                               index, 0, &cur_buf);
5816         if (ret) {
5817                 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5818                         __func__, ret);
5819                 return false;
5820         }
5821
5822         if (!cur_buf) {
5823                 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5824                          cur_buf);
5825                 return false;
5826         }
5827         /* Let it continue to flush when available buffer exceeds threshold */
5828         return avail_buf < hba->vps->wb_flush_threshold;
5829 }
5830
5831 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
5832 {
5833         if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
5834                 ufshcd_wb_toggle_flush(hba, false);
5835
5836         ufshcd_wb_toggle_flush_during_h8(hba, false);
5837         ufshcd_wb_toggle(hba, false);
5838         hba->caps &= ~UFSHCD_CAP_WB_EN;
5839
5840         dev_info(hba->dev, "%s: WB force disabled\n", __func__);
5841 }
5842
5843 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
5844 {
5845         u32 lifetime;
5846         int ret;
5847         u8 index;
5848
5849         index = ufshcd_wb_get_query_index(hba);
5850         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5851                                       QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
5852                                       index, 0, &lifetime);
5853         if (ret) {
5854                 dev_err(hba->dev,
5855                         "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
5856                         __func__, ret);
5857                 return false;
5858         }
5859
5860         if (lifetime == UFS_WB_EXCEED_LIFETIME) {
5861                 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
5862                         __func__, lifetime);
5863                 return false;
5864         }
5865
5866         dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
5867                 __func__, lifetime);
5868
5869         return true;
5870 }
5871
5872 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
5873 {
5874         int ret;
5875         u32 avail_buf;
5876         u8 index;
5877
5878         if (!ufshcd_is_wb_allowed(hba))
5879                 return false;
5880
5881         if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
5882                 ufshcd_wb_force_disable(hba);
5883                 return false;
5884         }
5885
5886         /*
5887          * The ufs device needs the vcc to be ON to flush.
5888          * With user-space reduction enabled, it's enough to enable flush
5889          * by checking only the available buffer. The threshold
5890          * defined here is > 90% full.
5891          * With user-space preserved enabled, the current-buffer
5892          * should be checked too because the wb buffer size can reduce
5893          * when disk tends to be full. This info is provided by current
5894          * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5895          * keeping vcc on when current buffer is empty.
5896          */
5897         index = ufshcd_wb_get_query_index(hba);
5898         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5899                                       QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
5900                                       index, 0, &avail_buf);
5901         if (ret) {
5902                 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5903                          __func__, ret);
5904                 return false;
5905         }
5906
5907         if (!hba->dev_info.b_presrv_uspc_en)
5908                 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
5909
5910         return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5911 }
5912
5913 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5914 {
5915         struct ufs_hba *hba = container_of(to_delayed_work(work),
5916                                            struct ufs_hba,
5917                                            rpm_dev_flush_recheck_work);
5918         /*
5919          * To prevent unnecessary VCC power drain after device finishes
5920          * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5921          * after a certain delay to recheck the threshold by next runtime
5922          * suspend.
5923          */
5924         ufshcd_rpm_get_sync(hba);
5925         ufshcd_rpm_put_sync(hba);
5926 }
5927
5928 /**
5929  * ufshcd_exception_event_handler - handle exceptions raised by device
5930  * @work: pointer to work data
5931  *
5932  * Read bExceptionEventStatus attribute from the device and handle the
5933  * exception event accordingly.
5934  */
5935 static void ufshcd_exception_event_handler(struct work_struct *work)
5936 {
5937         struct ufs_hba *hba;
5938         int err;
5939         u32 status = 0;
5940         hba = container_of(work, struct ufs_hba, eeh_work);
5941
5942         ufshcd_scsi_block_requests(hba);
5943         err = ufshcd_get_ee_status(hba, &status);
5944         if (err) {
5945                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5946                                 __func__, err);
5947                 goto out;
5948         }
5949
5950         trace_ufshcd_exception_event(dev_name(hba->dev), status);
5951
5952         if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
5953                 ufshcd_bkops_exception_event_handler(hba);
5954
5955         if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
5956                 ufshcd_temp_exception_event_handler(hba, status);
5957
5958         ufs_debugfs_exception_event(hba, status);
5959 out:
5960         ufshcd_scsi_unblock_requests(hba);
5961 }
5962
5963 /* Complete requests that have door-bell cleared */
5964 static void ufshcd_complete_requests(struct ufs_hba *hba)
5965 {
5966         ufshcd_transfer_req_compl(hba);
5967         ufshcd_tmc_handler(hba);
5968 }
5969
5970 /**
5971  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5972  *                              to recover from the DL NAC errors or not.
5973  * @hba: per-adapter instance
5974  *
5975  * Returns true if error handling is required, false otherwise
5976  */
5977 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5978 {
5979         unsigned long flags;
5980         bool err_handling = true;
5981
5982         spin_lock_irqsave(hba->host->host_lock, flags);
5983         /*
5984          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5985          * device fatal error and/or DL NAC & REPLAY timeout errors.
5986          */
5987         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5988                 goto out;
5989
5990         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5991             ((hba->saved_err & UIC_ERROR) &&
5992              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5993                 goto out;
5994
5995         if ((hba->saved_err & UIC_ERROR) &&
5996             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5997                 int err;
5998                 /*
5999                  * wait for 50ms to see if we can get any other errors or not.
6000                  */
6001                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6002                 msleep(50);
6003                 spin_lock_irqsave(hba->host->host_lock, flags);
6004
6005                 /*
6006                  * now check if we have got any other severe errors other than
6007                  * DL NAC error?
6008                  */
6009                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
6010                     ((hba->saved_err & UIC_ERROR) &&
6011                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6012                         goto out;
6013
6014                 /*
6015                  * As DL NAC is the only error received so far, send out NOP
6016                  * command to confirm if link is still active or not.
6017                  *   - If we don't get any response then do error recovery.
6018                  *   - If we get response then clear the DL NAC error bit.
6019                  */
6020
6021                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6022                 err = ufshcd_verify_dev_init(hba);
6023                 spin_lock_irqsave(hba->host->host_lock, flags);
6024
6025                 if (err)
6026                         goto out;
6027
6028                 /* Link seems to be alive hence ignore the DL NAC errors */
6029                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6030                         hba->saved_err &= ~UIC_ERROR;
6031                 /* clear NAC error */
6032                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6033                 if (!hba->saved_uic_err)
6034                         err_handling = false;
6035         }
6036 out:
6037         spin_unlock_irqrestore(hba->host->host_lock, flags);
6038         return err_handling;
6039 }
6040
6041 /* host lock must be held before calling this func */
6042 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6043 {
6044         return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6045                (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6046 }
6047
6048 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6049 {
6050         lockdep_assert_held(hba->host->host_lock);
6051
6052         /* handle fatal errors only when link is not in error state */
6053         if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6054                 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6055                     ufshcd_is_saved_err_fatal(hba))
6056                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6057                 else
6058                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6059                 queue_work(hba->eh_wq, &hba->eh_work);
6060         }
6061 }
6062
6063 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6064 {
6065         down_write(&hba->clk_scaling_lock);
6066         hba->clk_scaling.is_allowed = allow;
6067         up_write(&hba->clk_scaling_lock);
6068 }
6069
6070 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6071 {
6072         if (suspend) {
6073                 if (hba->clk_scaling.is_enabled)
6074                         ufshcd_suspend_clkscaling(hba);
6075                 ufshcd_clk_scaling_allow(hba, false);
6076         } else {
6077                 ufshcd_clk_scaling_allow(hba, true);
6078                 if (hba->clk_scaling.is_enabled)
6079                         ufshcd_resume_clkscaling(hba);
6080         }
6081 }
6082
6083 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6084 {
6085         ufshcd_rpm_get_sync(hba);
6086         if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6087             hba->is_sys_suspended) {
6088                 enum ufs_pm_op pm_op;
6089
6090                 /*
6091                  * Don't assume anything of resume, if
6092                  * resume fails, irq and clocks can be OFF, and powers
6093                  * can be OFF or in LPM.
6094                  */
6095                 ufshcd_setup_hba_vreg(hba, true);
6096                 ufshcd_enable_irq(hba);
6097                 ufshcd_setup_vreg(hba, true);
6098                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6099                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6100                 ufshcd_hold(hba, false);
6101                 if (!ufshcd_is_clkgating_allowed(hba))
6102                         ufshcd_setup_clocks(hba, true);
6103                 ufshcd_release(hba);
6104                 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6105                 ufshcd_vops_resume(hba, pm_op);
6106         } else {
6107                 ufshcd_hold(hba, false);
6108                 if (ufshcd_is_clkscaling_supported(hba) &&
6109                     hba->clk_scaling.is_enabled)
6110                         ufshcd_suspend_clkscaling(hba);
6111                 ufshcd_clk_scaling_allow(hba, false);
6112         }
6113         ufshcd_scsi_block_requests(hba);
6114         /* Drain ufshcd_queuecommand() */
6115         synchronize_rcu();
6116         cancel_work_sync(&hba->eeh_work);
6117 }
6118
6119 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6120 {
6121         ufshcd_scsi_unblock_requests(hba);
6122         ufshcd_release(hba);
6123         if (ufshcd_is_clkscaling_supported(hba))
6124                 ufshcd_clk_scaling_suspend(hba, false);
6125         ufshcd_rpm_put(hba);
6126 }
6127
6128 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6129 {
6130         return (!hba->is_powered || hba->shutting_down ||
6131                 !hba->ufs_device_wlun ||
6132                 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6133                 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6134                    ufshcd_is_link_broken(hba))));
6135 }
6136
6137 #ifdef CONFIG_PM
6138 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6139 {
6140         struct Scsi_Host *shost = hba->host;
6141         struct scsi_device *sdev;
6142         struct request_queue *q;
6143         int ret;
6144
6145         hba->is_sys_suspended = false;
6146         /*
6147          * Set RPM status of wlun device to RPM_ACTIVE,
6148          * this also clears its runtime error.
6149          */
6150         ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6151
6152         /* hba device might have a runtime error otherwise */
6153         if (ret)
6154                 ret = pm_runtime_set_active(hba->dev);
6155         /*
6156          * If wlun device had runtime error, we also need to resume those
6157          * consumer scsi devices in case any of them has failed to be
6158          * resumed due to supplier runtime resume failure. This is to unblock
6159          * blk_queue_enter in case there are bios waiting inside it.
6160          */
6161         if (!ret) {
6162                 shost_for_each_device(sdev, shost) {
6163                         q = sdev->request_queue;
6164                         if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6165                                        q->rpm_status == RPM_SUSPENDING))
6166                                 pm_request_resume(q->dev);
6167                 }
6168         }
6169 }
6170 #else
6171 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6172 {
6173 }
6174 #endif
6175
6176 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6177 {
6178         struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6179         u32 mode;
6180
6181         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6182
6183         if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6184                 return true;
6185
6186         if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6187                 return true;
6188
6189         return false;
6190 }
6191
6192 /**
6193  * ufshcd_err_handler - handle UFS errors that require s/w attention
6194  * @work: pointer to work structure
6195  */
6196 static void ufshcd_err_handler(struct work_struct *work)
6197 {
6198         int retries = MAX_ERR_HANDLER_RETRIES;
6199         struct ufs_hba *hba;
6200         unsigned long flags;
6201         bool needs_restore;
6202         bool needs_reset;
6203         bool err_xfer;
6204         bool err_tm;
6205         int pmc_err;
6206         int tag;
6207
6208         hba = container_of(work, struct ufs_hba, eh_work);
6209
6210         dev_info(hba->dev,
6211                  "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6212                  __func__, ufshcd_state_name[hba->ufshcd_state],
6213                  hba->is_powered, hba->shutting_down, hba->saved_err,
6214                  hba->saved_uic_err, hba->force_reset,
6215                  ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6216
6217         down(&hba->host_sem);
6218         spin_lock_irqsave(hba->host->host_lock, flags);
6219         if (ufshcd_err_handling_should_stop(hba)) {
6220                 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6221                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6222                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6223                 up(&hba->host_sem);
6224                 return;
6225         }
6226         ufshcd_set_eh_in_progress(hba);
6227         spin_unlock_irqrestore(hba->host->host_lock, flags);
6228         ufshcd_err_handling_prepare(hba);
6229         /* Complete requests that have door-bell cleared by h/w */
6230         ufshcd_complete_requests(hba);
6231         spin_lock_irqsave(hba->host->host_lock, flags);
6232 again:
6233         needs_restore = false;
6234         needs_reset = false;
6235         err_xfer = false;
6236         err_tm = false;
6237
6238         if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6239                 hba->ufshcd_state = UFSHCD_STATE_RESET;
6240         /*
6241          * A full reset and restore might have happened after preparation
6242          * is finished, double check whether we should stop.
6243          */
6244         if (ufshcd_err_handling_should_stop(hba))
6245                 goto skip_err_handling;
6246
6247         if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6248                 bool ret;
6249
6250                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6251                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6252                 ret = ufshcd_quirk_dl_nac_errors(hba);
6253                 spin_lock_irqsave(hba->host->host_lock, flags);
6254                 if (!ret && ufshcd_err_handling_should_stop(hba))
6255                         goto skip_err_handling;
6256         }
6257
6258         if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6259             (hba->saved_uic_err &&
6260              (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6261                 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6262
6263                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6264                 ufshcd_print_host_state(hba);
6265                 ufshcd_print_pwr_info(hba);
6266                 ufshcd_print_evt_hist(hba);
6267                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6268                 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
6269                 spin_lock_irqsave(hba->host->host_lock, flags);
6270         }
6271
6272         /*
6273          * if host reset is required then skip clearing the pending
6274          * transfers forcefully because they will get cleared during
6275          * host reset and restore
6276          */
6277         if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6278             ufshcd_is_saved_err_fatal(hba) ||
6279             ((hba->saved_err & UIC_ERROR) &&
6280              (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6281                                     UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6282                 needs_reset = true;
6283                 goto do_reset;
6284         }
6285
6286         /*
6287          * If LINERESET was caught, UFS might have been put to PWM mode,
6288          * check if power mode restore is needed.
6289          */
6290         if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6291                 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6292                 if (!hba->saved_uic_err)
6293                         hba->saved_err &= ~UIC_ERROR;
6294                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6295                 if (ufshcd_is_pwr_mode_restore_needed(hba))
6296                         needs_restore = true;
6297                 spin_lock_irqsave(hba->host->host_lock, flags);
6298                 if (!hba->saved_err && !needs_restore)
6299                         goto skip_err_handling;
6300         }
6301
6302         hba->silence_err_logs = true;
6303         /* release lock as clear command might sleep */
6304         spin_unlock_irqrestore(hba->host->host_lock, flags);
6305         /* Clear pending transfer requests */
6306         for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
6307                 if (ufshcd_try_to_abort_task(hba, tag)) {
6308                         err_xfer = true;
6309                         goto lock_skip_pending_xfer_clear;
6310                 }
6311                 dev_err(hba->dev, "Aborted tag %d / CDB %#02x\n", tag,
6312                         hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1);
6313         }
6314
6315         /* Clear pending task management requests */
6316         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6317                 if (ufshcd_clear_tm_cmd(hba, tag)) {
6318                         err_tm = true;
6319                         goto lock_skip_pending_xfer_clear;
6320                 }
6321         }
6322
6323 lock_skip_pending_xfer_clear:
6324         /* Complete the requests that are cleared by s/w */
6325         ufshcd_complete_requests(hba);
6326
6327         spin_lock_irqsave(hba->host->host_lock, flags);
6328         hba->silence_err_logs = false;
6329         if (err_xfer || err_tm) {
6330                 needs_reset = true;
6331                 goto do_reset;
6332         }
6333
6334         /*
6335          * After all reqs and tasks are cleared from doorbell,
6336          * now it is safe to retore power mode.
6337          */
6338         if (needs_restore) {
6339                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6340                 /*
6341                  * Hold the scaling lock just in case dev cmds
6342                  * are sent via bsg and/or sysfs.
6343                  */
6344                 down_write(&hba->clk_scaling_lock);
6345                 hba->force_pmc = true;
6346                 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6347                 if (pmc_err) {
6348                         needs_reset = true;
6349                         dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6350                                         __func__, pmc_err);
6351                 }
6352                 hba->force_pmc = false;
6353                 ufshcd_print_pwr_info(hba);
6354                 up_write(&hba->clk_scaling_lock);
6355                 spin_lock_irqsave(hba->host->host_lock, flags);
6356         }
6357
6358 do_reset:
6359         /* Fatal errors need reset */
6360         if (needs_reset) {
6361                 int err;
6362
6363                 hba->force_reset = false;
6364                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6365                 err = ufshcd_reset_and_restore(hba);
6366                 if (err)
6367                         dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6368                                         __func__, err);
6369                 else
6370                         ufshcd_recover_pm_error(hba);
6371                 spin_lock_irqsave(hba->host->host_lock, flags);
6372         }
6373
6374 skip_err_handling:
6375         if (!needs_reset) {
6376                 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6377                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6378                 if (hba->saved_err || hba->saved_uic_err)
6379                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6380                             __func__, hba->saved_err, hba->saved_uic_err);
6381         }
6382         /* Exit in an operational state or dead */
6383         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6384             hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6385                 if (--retries)
6386                         goto again;
6387                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6388         }
6389         ufshcd_clear_eh_in_progress(hba);
6390         spin_unlock_irqrestore(hba->host->host_lock, flags);
6391         ufshcd_err_handling_unprepare(hba);
6392         up(&hba->host_sem);
6393
6394         dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6395                  ufshcd_state_name[hba->ufshcd_state]);
6396 }
6397
6398 /**
6399  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6400  * @hba: per-adapter instance
6401  *
6402  * Returns
6403  *  IRQ_HANDLED - If interrupt is valid
6404  *  IRQ_NONE    - If invalid interrupt
6405  */
6406 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6407 {
6408         u32 reg;
6409         irqreturn_t retval = IRQ_NONE;
6410
6411         /* PHY layer error */
6412         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6413         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6414             (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6415                 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6416                 /*
6417                  * To know whether this error is fatal or not, DB timeout
6418                  * must be checked but this error is handled separately.
6419                  */
6420                 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6421                         dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6422                                         __func__);
6423
6424                 /* Got a LINERESET indication. */
6425                 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6426                         struct uic_command *cmd = NULL;
6427
6428                         hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6429                         if (hba->uic_async_done && hba->active_uic_cmd)
6430                                 cmd = hba->active_uic_cmd;
6431                         /*
6432                          * Ignore the LINERESET during power mode change
6433                          * operation via DME_SET command.
6434                          */
6435                         if (cmd && (cmd->command == UIC_CMD_DME_SET))
6436                                 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6437                 }
6438                 retval |= IRQ_HANDLED;
6439         }
6440
6441         /* PA_INIT_ERROR is fatal and needs UIC reset */
6442         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6443         if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6444             (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6445                 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6446
6447                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6448                         hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6449                 else if (hba->dev_quirks &
6450                                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6451                         if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6452                                 hba->uic_error |=
6453                                         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6454                         else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6455                                 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6456                 }
6457                 retval |= IRQ_HANDLED;
6458         }
6459
6460         /* UIC NL/TL/DME errors needs software retry */
6461         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6462         if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6463             (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6464                 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6465                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6466                 retval |= IRQ_HANDLED;
6467         }
6468
6469         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6470         if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6471             (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6472                 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6473                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6474                 retval |= IRQ_HANDLED;
6475         }
6476
6477         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6478         if ((reg & UIC_DME_ERROR) &&
6479             (reg & UIC_DME_ERROR_CODE_MASK)) {
6480                 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6481                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6482                 retval |= IRQ_HANDLED;
6483         }
6484
6485         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6486                         __func__, hba->uic_error);
6487         return retval;
6488 }
6489
6490 /**
6491  * ufshcd_check_errors - Check for errors that need s/w attention
6492  * @hba: per-adapter instance
6493  * @intr_status: interrupt status generated by the controller
6494  *
6495  * Returns
6496  *  IRQ_HANDLED - If interrupt is valid
6497  *  IRQ_NONE    - If invalid interrupt
6498  */
6499 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6500 {
6501         bool queue_eh_work = false;
6502         irqreturn_t retval = IRQ_NONE;
6503
6504         spin_lock(hba->host->host_lock);
6505         hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6506
6507         if (hba->errors & INT_FATAL_ERRORS) {
6508                 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6509                                        hba->errors);
6510                 queue_eh_work = true;
6511         }
6512
6513         if (hba->errors & UIC_ERROR) {
6514                 hba->uic_error = 0;
6515                 retval = ufshcd_update_uic_error(hba);
6516                 if (hba->uic_error)
6517                         queue_eh_work = true;
6518         }
6519
6520         if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6521                 dev_err(hba->dev,
6522                         "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6523                         __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6524                         "Enter" : "Exit",
6525                         hba->errors, ufshcd_get_upmcrs(hba));
6526                 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6527                                        hba->errors);
6528                 ufshcd_set_link_broken(hba);
6529                 queue_eh_work = true;
6530         }
6531
6532         if (queue_eh_work) {
6533                 /*
6534                  * update the transfer error masks to sticky bits, let's do this
6535                  * irrespective of current ufshcd_state.
6536                  */
6537                 hba->saved_err |= hba->errors;
6538                 hba->saved_uic_err |= hba->uic_error;
6539
6540                 /* dump controller state before resetting */
6541                 if ((hba->saved_err &
6542                      (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6543                     (hba->saved_uic_err &&
6544                      (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6545                         dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6546                                         __func__, hba->saved_err,
6547                                         hba->saved_uic_err);
6548                         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6549                                          "host_regs: ");
6550                         ufshcd_print_pwr_info(hba);
6551                 }
6552                 ufshcd_schedule_eh_work(hba);
6553                 retval |= IRQ_HANDLED;
6554         }
6555         /*
6556          * if (!queue_eh_work) -
6557          * Other errors are either non-fatal where host recovers
6558          * itself without s/w intervention or errors that will be
6559          * handled by the SCSI core layer.
6560          */
6561         hba->errors = 0;
6562         hba->uic_error = 0;
6563         spin_unlock(hba->host->host_lock);
6564         return retval;
6565 }
6566
6567 /**
6568  * ufshcd_tmc_handler - handle task management function completion
6569  * @hba: per adapter instance
6570  *
6571  * Returns
6572  *  IRQ_HANDLED - If interrupt is valid
6573  *  IRQ_NONE    - If invalid interrupt
6574  */
6575 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6576 {
6577         unsigned long flags, pending, issued;
6578         irqreturn_t ret = IRQ_NONE;
6579         int tag;
6580
6581         spin_lock_irqsave(hba->host->host_lock, flags);
6582         pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6583         issued = hba->outstanding_tasks & ~pending;
6584         for_each_set_bit(tag, &issued, hba->nutmrs) {
6585                 struct request *req = hba->tmf_rqs[tag];
6586                 struct completion *c = req->end_io_data;
6587
6588                 complete(c);
6589                 ret = IRQ_HANDLED;
6590         }
6591         spin_unlock_irqrestore(hba->host->host_lock, flags);
6592
6593         return ret;
6594 }
6595
6596 /**
6597  * ufshcd_sl_intr - Interrupt service routine
6598  * @hba: per adapter instance
6599  * @intr_status: contains interrupts generated by the controller
6600  *
6601  * Returns
6602  *  IRQ_HANDLED - If interrupt is valid
6603  *  IRQ_NONE    - If invalid interrupt
6604  */
6605 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6606 {
6607         irqreturn_t retval = IRQ_NONE;
6608
6609         if (intr_status & UFSHCD_UIC_MASK)
6610                 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6611
6612         if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6613                 retval |= ufshcd_check_errors(hba, intr_status);
6614
6615         if (intr_status & UTP_TASK_REQ_COMPL)
6616                 retval |= ufshcd_tmc_handler(hba);
6617
6618         if (intr_status & UTP_TRANSFER_REQ_COMPL)
6619                 retval |= ufshcd_transfer_req_compl(hba);
6620
6621         return retval;
6622 }
6623
6624 /**
6625  * ufshcd_intr - Main interrupt service routine
6626  * @irq: irq number
6627  * @__hba: pointer to adapter instance
6628  *
6629  * Returns
6630  *  IRQ_HANDLED - If interrupt is valid
6631  *  IRQ_NONE    - If invalid interrupt
6632  */
6633 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6634 {
6635         u32 intr_status, enabled_intr_status = 0;
6636         irqreturn_t retval = IRQ_NONE;
6637         struct ufs_hba *hba = __hba;
6638         int retries = hba->nutrs;
6639
6640         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6641         hba->ufs_stats.last_intr_status = intr_status;
6642         hba->ufs_stats.last_intr_ts = ktime_get();
6643
6644         /*
6645          * There could be max of hba->nutrs reqs in flight and in worst case
6646          * if the reqs get finished 1 by 1 after the interrupt status is
6647          * read, make sure we handle them by checking the interrupt status
6648          * again in a loop until we process all of the reqs before returning.
6649          */
6650         while (intr_status && retries--) {
6651                 enabled_intr_status =
6652                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6653                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6654                 if (enabled_intr_status)
6655                         retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6656
6657                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6658         }
6659
6660         if (enabled_intr_status && retval == IRQ_NONE &&
6661             (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6662              hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6663                 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6664                                         __func__,
6665                                         intr_status,
6666                                         hba->ufs_stats.last_intr_status,
6667                                         enabled_intr_status);
6668                 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6669         }
6670
6671         return retval;
6672 }
6673
6674 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6675 {
6676         int err = 0;
6677         u32 mask = 1 << tag;
6678         unsigned long flags;
6679
6680         if (!test_bit(tag, &hba->outstanding_tasks))
6681                 goto out;
6682
6683         spin_lock_irqsave(hba->host->host_lock, flags);
6684         ufshcd_utmrl_clear(hba, tag);
6685         spin_unlock_irqrestore(hba->host->host_lock, flags);
6686
6687         /* poll for max. 1 sec to clear door bell register by h/w */
6688         err = ufshcd_wait_for_register(hba,
6689                         REG_UTP_TASK_REQ_DOOR_BELL,
6690                         mask, 0, 1000, 1000);
6691
6692         dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
6693                 tag, err ? "succeeded" : "failed");
6694
6695 out:
6696         return err;
6697 }
6698
6699 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6700                 struct utp_task_req_desc *treq, u8 tm_function)
6701 {
6702         struct request_queue *q = hba->tmf_queue;
6703         struct Scsi_Host *host = hba->host;
6704         DECLARE_COMPLETION_ONSTACK(wait);
6705         struct request *req;
6706         unsigned long flags;
6707         int task_tag, err;
6708
6709         /*
6710          * blk_mq_alloc_request() is used here only to get a free tag.
6711          */
6712         req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
6713         if (IS_ERR(req))
6714                 return PTR_ERR(req);
6715
6716         req->end_io_data = &wait;
6717         ufshcd_hold(hba, false);
6718
6719         spin_lock_irqsave(host->host_lock, flags);
6720
6721         task_tag = req->tag;
6722         WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n",
6723                   task_tag);
6724         hba->tmf_rqs[req->tag] = req;
6725         treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag);
6726
6727         memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
6728         ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
6729
6730         /* send command to the controller */
6731         __set_bit(task_tag, &hba->outstanding_tasks);
6732
6733         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
6734         /* Make sure that doorbell is committed immediately */
6735         wmb();
6736
6737         spin_unlock_irqrestore(host->host_lock, flags);
6738
6739         ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
6740
6741         /* wait until the task management command is completed */
6742         err = wait_for_completion_io_timeout(&wait,
6743                         msecs_to_jiffies(TM_CMD_TIMEOUT));
6744         if (!err) {
6745                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
6746                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6747                                 __func__, tm_function);
6748                 if (ufshcd_clear_tm_cmd(hba, task_tag))
6749                         dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
6750                                         __func__, task_tag);
6751                 err = -ETIMEDOUT;
6752         } else {
6753                 err = 0;
6754                 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
6755
6756                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
6757         }
6758
6759         spin_lock_irqsave(hba->host->host_lock, flags);
6760         hba->tmf_rqs[req->tag] = NULL;
6761         __clear_bit(task_tag, &hba->outstanding_tasks);
6762         spin_unlock_irqrestore(hba->host->host_lock, flags);
6763
6764         ufshcd_release(hba);
6765         blk_mq_free_request(req);
6766
6767         return err;
6768 }
6769
6770 /**
6771  * ufshcd_issue_tm_cmd - issues task management commands to controller
6772  * @hba: per adapter instance
6773  * @lun_id: LUN ID to which TM command is sent
6774  * @task_id: task ID to which the TM command is applicable
6775  * @tm_function: task management function opcode
6776  * @tm_response: task management service response return value
6777  *
6778  * Returns non-zero value on error, zero on success.
6779  */
6780 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6781                 u8 tm_function, u8 *tm_response)
6782 {
6783         struct utp_task_req_desc treq = { { 0 }, };
6784         enum utp_ocs ocs_value;
6785         int err;
6786
6787         /* Configure task request descriptor */
6788         treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6789         treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6790
6791         /* Configure task request UPIU */
6792         treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6793                                   cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6794         treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6795
6796         /*
6797          * The host shall provide the same value for LUN field in the basic
6798          * header and for Input Parameter.
6799          */
6800         treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
6801         treq.upiu_req.input_param2 = cpu_to_be32(task_id);
6802
6803         err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6804         if (err == -ETIMEDOUT)
6805                 return err;
6806
6807         ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6808         if (ocs_value != OCS_SUCCESS)
6809                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6810                                 __func__, ocs_value);
6811         else if (tm_response)
6812                 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
6813                                 MASK_TM_SERVICE_RESP;
6814         return err;
6815 }
6816
6817 /**
6818  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6819  * @hba:        per-adapter instance
6820  * @req_upiu:   upiu request
6821  * @rsp_upiu:   upiu reply
6822  * @desc_buff:  pointer to descriptor buffer, NULL if NA
6823  * @buff_len:   descriptor size, 0 if NA
6824  * @cmd_type:   specifies the type (NOP, Query...)
6825  * @desc_op:    descriptor operation
6826  *
6827  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6828  * Therefore, it "rides" the device management infrastructure: uses its tag and
6829  * tasks work queues.
6830  *
6831  * Since there is only one available tag for device management commands,
6832  * the caller is expected to hold the hba->dev_cmd.lock mutex.
6833  */
6834 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6835                                         struct utp_upiu_req *req_upiu,
6836                                         struct utp_upiu_req *rsp_upiu,
6837                                         u8 *desc_buff, int *buff_len,
6838                                         enum dev_cmd_type cmd_type,
6839                                         enum query_opcode desc_op)
6840 {
6841         DECLARE_COMPLETION_ONSTACK(wait);
6842         const u32 tag = hba->reserved_slot;
6843         struct ufshcd_lrb *lrbp;
6844         int err = 0;
6845         u8 upiu_flags;
6846
6847         /* Protects use of hba->reserved_slot. */
6848         lockdep_assert_held(&hba->dev_cmd.lock);
6849
6850         down_read(&hba->clk_scaling_lock);
6851
6852         lrbp = &hba->lrb[tag];
6853         WARN_ON(lrbp->cmd);
6854         lrbp->cmd = NULL;
6855         lrbp->task_tag = tag;
6856         lrbp->lun = 0;
6857         lrbp->intr_cmd = true;
6858         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
6859         hba->dev_cmd.type = cmd_type;
6860
6861         if (hba->ufs_version <= ufshci_version(1, 1))
6862                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6863         else
6864                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6865
6866         /* update the task tag in the request upiu */
6867         req_upiu->header.dword_0 |= cpu_to_be32(tag);
6868
6869         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6870
6871         /* just copy the upiu request as it is */
6872         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6873         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6874                 /* The Data Segment Area is optional depending upon the query
6875                  * function value. for WRITE DESCRIPTOR, the data segment
6876                  * follows right after the tsf.
6877                  */
6878                 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6879                 *buff_len = 0;
6880         }
6881
6882         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6883
6884         hba->dev_cmd.complete = &wait;
6885
6886         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
6887
6888         ufshcd_send_command(hba, tag);
6889         /*
6890          * ignore the returning value here - ufshcd_check_query_response is
6891          * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6892          * read the response directly ignoring all errors.
6893          */
6894         ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6895
6896         /* just copy the upiu response as it is */
6897         memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
6898         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6899                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6900                 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6901                                MASK_QUERY_DATA_SEG_LEN;
6902
6903                 if (*buff_len >= resp_len) {
6904                         memcpy(desc_buff, descp, resp_len);
6905                         *buff_len = resp_len;
6906                 } else {
6907                         dev_warn(hba->dev,
6908                                  "%s: rsp size %d is bigger than buffer size %d",
6909                                  __func__, resp_len, *buff_len);
6910                         *buff_len = 0;
6911                         err = -EINVAL;
6912                 }
6913         }
6914         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
6915                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
6916
6917         up_read(&hba->clk_scaling_lock);
6918         return err;
6919 }
6920
6921 /**
6922  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6923  * @hba:        per-adapter instance
6924  * @req_upiu:   upiu request
6925  * @rsp_upiu:   upiu reply - only 8 DW as we do not support scsi commands
6926  * @msgcode:    message code, one of UPIU Transaction Codes Initiator to Target
6927  * @desc_buff:  pointer to descriptor buffer, NULL if NA
6928  * @buff_len:   descriptor size, 0 if NA
6929  * @desc_op:    descriptor operation
6930  *
6931  * Supports UTP Transfer requests (nop and query), and UTP Task
6932  * Management requests.
6933  * It is up to the caller to fill the upiu conent properly, as it will
6934  * be copied without any further input validations.
6935  */
6936 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6937                              struct utp_upiu_req *req_upiu,
6938                              struct utp_upiu_req *rsp_upiu,
6939                              int msgcode,
6940                              u8 *desc_buff, int *buff_len,
6941                              enum query_opcode desc_op)
6942 {
6943         int err;
6944         enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
6945         struct utp_task_req_desc treq = { { 0 }, };
6946         enum utp_ocs ocs_value;
6947         u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6948
6949         switch (msgcode) {
6950         case UPIU_TRANSACTION_NOP_OUT:
6951                 cmd_type = DEV_CMD_TYPE_NOP;
6952                 fallthrough;
6953         case UPIU_TRANSACTION_QUERY_REQ:
6954                 ufshcd_hold(hba, false);
6955                 mutex_lock(&hba->dev_cmd.lock);
6956                 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6957                                                    desc_buff, buff_len,
6958                                                    cmd_type, desc_op);
6959                 mutex_unlock(&hba->dev_cmd.lock);
6960                 ufshcd_release(hba);
6961
6962                 break;
6963         case UPIU_TRANSACTION_TASK_REQ:
6964                 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6965                 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6966
6967                 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
6968
6969                 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6970                 if (err == -ETIMEDOUT)
6971                         break;
6972
6973                 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6974                 if (ocs_value != OCS_SUCCESS) {
6975                         dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6976                                 ocs_value);
6977                         break;
6978                 }
6979
6980                 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
6981
6982                 break;
6983         default:
6984                 err = -EINVAL;
6985
6986                 break;
6987         }
6988
6989         return err;
6990 }
6991
6992 /**
6993  * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
6994  * @cmd: SCSI command pointer
6995  *
6996  * Returns SUCCESS/FAILED
6997  */
6998 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
6999 {
7000         unsigned long flags, pending_reqs = 0, not_cleared = 0;
7001         struct Scsi_Host *host;
7002         struct ufs_hba *hba;
7003         u32 pos;
7004         int err;
7005         u8 resp = 0xF, lun;
7006
7007         host = cmd->device->host;
7008         hba = shost_priv(host);
7009
7010         lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7011         err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7012         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7013                 if (!err)
7014                         err = resp;
7015                 goto out;
7016         }
7017
7018         /* clear the commands that were pending for corresponding LUN */
7019         spin_lock_irqsave(&hba->outstanding_lock, flags);
7020         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7021                 if (hba->lrb[pos].lun == lun)
7022                         __set_bit(pos, &pending_reqs);
7023         hba->outstanding_reqs &= ~pending_reqs;
7024         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7025
7026         if (ufshcd_clear_cmds(hba, pending_reqs) < 0) {
7027                 spin_lock_irqsave(&hba->outstanding_lock, flags);
7028                 not_cleared = pending_reqs &
7029                         ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7030                 hba->outstanding_reqs |= not_cleared;
7031                 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7032
7033                 dev_err(hba->dev, "%s: failed to clear requests %#lx\n",
7034                         __func__, not_cleared);
7035         }
7036         __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared);
7037
7038 out:
7039         hba->req_abort_count = 0;
7040         ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7041         if (!err) {
7042                 err = SUCCESS;
7043         } else {
7044                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7045                 err = FAILED;
7046         }
7047         return err;
7048 }
7049
7050 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7051 {
7052         struct ufshcd_lrb *lrbp;
7053         int tag;
7054
7055         for_each_set_bit(tag, &bitmap, hba->nutrs) {
7056                 lrbp = &hba->lrb[tag];
7057                 lrbp->req_abort_skip = true;
7058         }
7059 }
7060
7061 /**
7062  * ufshcd_try_to_abort_task - abort a specific task
7063  * @hba: Pointer to adapter instance
7064  * @tag: Task tag/index to be aborted
7065  *
7066  * Abort the pending command in device by sending UFS_ABORT_TASK task management
7067  * command, and in host controller by clearing the door-bell register. There can
7068  * be race between controller sending the command to the device while abort is
7069  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7070  * really issued and then try to abort it.
7071  *
7072  * Returns zero on success, non-zero on failure
7073  */
7074 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7075 {
7076         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7077         int err = 0;
7078         int poll_cnt;
7079         u8 resp = 0xF;
7080         u32 reg;
7081
7082         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7083                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7084                                 UFS_QUERY_TASK, &resp);
7085                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7086                         /* cmd pending in the device */
7087                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7088                                 __func__, tag);
7089                         break;
7090                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7091                         /*
7092                          * cmd not pending in the device, check if it is
7093                          * in transition.
7094                          */
7095                         dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7096                                 __func__, tag);
7097                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7098                         if (reg & (1 << tag)) {
7099                                 /* sleep for max. 200us to stabilize */
7100                                 usleep_range(100, 200);
7101                                 continue;
7102                         }
7103                         /* command completed already */
7104                         dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7105                                 __func__, tag);
7106                         goto out;
7107                 } else {
7108                         dev_err(hba->dev,
7109                                 "%s: no response from device. tag = %d, err %d\n",
7110                                 __func__, tag, err);
7111                         if (!err)
7112                                 err = resp; /* service response error */
7113                         goto out;
7114                 }
7115         }
7116
7117         if (!poll_cnt) {
7118                 err = -EBUSY;
7119                 goto out;
7120         }
7121
7122         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7123                         UFS_ABORT_TASK, &resp);
7124         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7125                 if (!err) {
7126                         err = resp; /* service response error */
7127                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7128                                 __func__, tag, err);
7129                 }
7130                 goto out;
7131         }
7132
7133         err = ufshcd_clear_cmds(hba, 1U << tag);
7134         if (err)
7135                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7136                         __func__, tag, err);
7137
7138 out:
7139         return err;
7140 }
7141
7142 /**
7143  * ufshcd_abort - scsi host template eh_abort_handler callback
7144  * @cmd: SCSI command pointer
7145  *
7146  * Returns SUCCESS/FAILED
7147  */
7148 static int ufshcd_abort(struct scsi_cmnd *cmd)
7149 {
7150         struct Scsi_Host *host = cmd->device->host;
7151         struct ufs_hba *hba = shost_priv(host);
7152         int tag = scsi_cmd_to_rq(cmd)->tag;
7153         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7154         unsigned long flags;
7155         int err = FAILED;
7156         bool outstanding;
7157         u32 reg;
7158
7159         WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
7160
7161         ufshcd_hold(hba, false);
7162         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7163         /* If command is already aborted/completed, return FAILED. */
7164         if (!(test_bit(tag, &hba->outstanding_reqs))) {
7165                 dev_err(hba->dev,
7166                         "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7167                         __func__, tag, hba->outstanding_reqs, reg);
7168                 goto release;
7169         }
7170
7171         /* Print Transfer Request of aborted task */
7172         dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7173
7174         /*
7175          * Print detailed info about aborted request.
7176          * As more than one request might get aborted at the same time,
7177          * print full information only for the first aborted request in order
7178          * to reduce repeated printouts. For other aborted requests only print
7179          * basic details.
7180          */
7181         scsi_print_command(cmd);
7182         if (!hba->req_abort_count) {
7183                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7184                 ufshcd_print_evt_hist(hba);
7185                 ufshcd_print_host_state(hba);
7186                 ufshcd_print_pwr_info(hba);
7187                 ufshcd_print_trs(hba, 1 << tag, true);
7188         } else {
7189                 ufshcd_print_trs(hba, 1 << tag, false);
7190         }
7191         hba->req_abort_count++;
7192
7193         if (!(reg & (1 << tag))) {
7194                 dev_err(hba->dev,
7195                 "%s: cmd was completed, but without a notifying intr, tag = %d",
7196                 __func__, tag);
7197                 __ufshcd_transfer_req_compl(hba, 1UL << tag);
7198                 goto release;
7199         }
7200
7201         /*
7202          * Task abort to the device W-LUN is illegal. When this command
7203          * will fail, due to spec violation, scsi err handling next step
7204          * will be to send LU reset which, again, is a spec violation.
7205          * To avoid these unnecessary/illegal steps, first we clean up
7206          * the lrb taken by this cmd and re-set it in outstanding_reqs,
7207          * then queue the eh_work and bail.
7208          */
7209         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7210                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7211
7212                 spin_lock_irqsave(host->host_lock, flags);
7213                 hba->force_reset = true;
7214                 ufshcd_schedule_eh_work(hba);
7215                 spin_unlock_irqrestore(host->host_lock, flags);
7216                 goto release;
7217         }
7218
7219         /* Skip task abort in case previous aborts failed and report failure */
7220         if (lrbp->req_abort_skip) {
7221                 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7222                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7223                 goto release;
7224         }
7225
7226         err = ufshcd_try_to_abort_task(hba, tag);
7227         if (err) {
7228                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7229                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7230                 err = FAILED;
7231                 goto release;
7232         }
7233
7234         /*
7235          * Clear the corresponding bit from outstanding_reqs since the command
7236          * has been aborted successfully.
7237          */
7238         spin_lock_irqsave(&hba->outstanding_lock, flags);
7239         outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7240         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7241
7242         if (outstanding)
7243                 ufshcd_release_scsi_cmd(hba, lrbp);
7244
7245         err = SUCCESS;
7246
7247 release:
7248         /* Matches the ufshcd_hold() call at the start of this function. */
7249         ufshcd_release(hba);
7250         return err;
7251 }
7252
7253 /**
7254  * ufshcd_host_reset_and_restore - reset and restore host controller
7255  * @hba: per-adapter instance
7256  *
7257  * Note that host controller reset may issue DME_RESET to
7258  * local and remote (device) Uni-Pro stack and the attributes
7259  * are reset to default state.
7260  *
7261  * Returns zero on success, non-zero on failure
7262  */
7263 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7264 {
7265         int err;
7266
7267         /*
7268          * Stop the host controller and complete the requests
7269          * cleared by h/w
7270          */
7271         ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET);
7272         ufshcd_hba_stop(hba);
7273         hba->silence_err_logs = true;
7274         ufshcd_complete_requests(hba);
7275         hba->silence_err_logs = false;
7276
7277         /* scale up clocks to max frequency before full reinitialization */
7278         ufshcd_scale_clks(hba, true);
7279
7280         err = ufshcd_hba_enable(hba);
7281
7282         /* Establish the link again and restore the device */
7283         if (!err)
7284                 err = ufshcd_probe_hba(hba, false);
7285
7286         if (err)
7287                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7288         ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7289         return err;
7290 }
7291
7292 /**
7293  * ufshcd_reset_and_restore - reset and re-initialize host/device
7294  * @hba: per-adapter instance
7295  *
7296  * Reset and recover device, host and re-establish link. This
7297  * is helpful to recover the communication in fatal error conditions.
7298  *
7299  * Returns zero on success, non-zero on failure
7300  */
7301 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7302 {
7303         u32 saved_err = 0;
7304         u32 saved_uic_err = 0;
7305         int err = 0;
7306         unsigned long flags;
7307         int retries = MAX_HOST_RESET_RETRIES;
7308
7309         spin_lock_irqsave(hba->host->host_lock, flags);
7310         do {
7311                 /*
7312                  * This is a fresh start, cache and clear saved error first,
7313                  * in case new error generated during reset and restore.
7314                  */
7315                 saved_err |= hba->saved_err;
7316                 saved_uic_err |= hba->saved_uic_err;
7317                 hba->saved_err = 0;
7318                 hba->saved_uic_err = 0;
7319                 hba->force_reset = false;
7320                 hba->ufshcd_state = UFSHCD_STATE_RESET;
7321                 spin_unlock_irqrestore(hba->host->host_lock, flags);
7322
7323                 /* Reset the attached device */
7324                 ufshcd_device_reset(hba);
7325
7326                 err = ufshcd_host_reset_and_restore(hba);
7327
7328                 spin_lock_irqsave(hba->host->host_lock, flags);
7329                 if (err)
7330                         continue;
7331                 /* Do not exit unless operational or dead */
7332                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7333                     hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7334                     hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7335                         err = -EAGAIN;
7336         } while (err && --retries);
7337
7338         /*
7339          * Inform scsi mid-layer that we did reset and allow to handle
7340          * Unit Attention properly.
7341          */
7342         scsi_report_bus_reset(hba->host, 0);
7343         if (err) {
7344                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7345                 hba->saved_err |= saved_err;
7346                 hba->saved_uic_err |= saved_uic_err;
7347         }
7348         spin_unlock_irqrestore(hba->host->host_lock, flags);
7349
7350         return err;
7351 }
7352
7353 /**
7354  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7355  * @cmd: SCSI command pointer
7356  *
7357  * Returns SUCCESS/FAILED
7358  */
7359 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7360 {
7361         int err = SUCCESS;
7362         unsigned long flags;
7363         struct ufs_hba *hba;
7364
7365         hba = shost_priv(cmd->device->host);
7366
7367         spin_lock_irqsave(hba->host->host_lock, flags);
7368         hba->force_reset = true;
7369         ufshcd_schedule_eh_work(hba);
7370         dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7371         spin_unlock_irqrestore(hba->host->host_lock, flags);
7372
7373         flush_work(&hba->eh_work);
7374
7375         spin_lock_irqsave(hba->host->host_lock, flags);
7376         if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7377                 err = FAILED;
7378         spin_unlock_irqrestore(hba->host->host_lock, flags);
7379
7380         return err;
7381 }
7382
7383 /**
7384  * ufshcd_get_max_icc_level - calculate the ICC level
7385  * @sup_curr_uA: max. current supported by the regulator
7386  * @start_scan: row at the desc table to start scan from
7387  * @buff: power descriptor buffer
7388  *
7389  * Returns calculated max ICC level for specific regulator
7390  */
7391 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
7392 {
7393         int i;
7394         int curr_uA;
7395         u16 data;
7396         u16 unit;
7397
7398         for (i = start_scan; i >= 0; i--) {
7399                 data = get_unaligned_be16(&buff[2 * i]);
7400                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7401                                                 ATTR_ICC_LVL_UNIT_OFFSET;
7402                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7403                 switch (unit) {
7404                 case UFSHCD_NANO_AMP:
7405                         curr_uA = curr_uA / 1000;
7406                         break;
7407                 case UFSHCD_MILI_AMP:
7408                         curr_uA = curr_uA * 1000;
7409                         break;
7410                 case UFSHCD_AMP:
7411                         curr_uA = curr_uA * 1000 * 1000;
7412                         break;
7413                 case UFSHCD_MICRO_AMP:
7414                 default:
7415                         break;
7416                 }
7417                 if (sup_curr_uA >= curr_uA)
7418                         break;
7419         }
7420         if (i < 0) {
7421                 i = 0;
7422                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7423         }
7424
7425         return (u32)i;
7426 }
7427
7428 /**
7429  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7430  * In case regulators are not initialized we'll return 0
7431  * @hba: per-adapter instance
7432  * @desc_buf: power descriptor buffer to extract ICC levels from.
7433  * @len: length of desc_buff
7434  *
7435  * Returns calculated ICC level
7436  */
7437 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7438                                                         u8 *desc_buf, int len)
7439 {
7440         u32 icc_level = 0;
7441
7442         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7443                                                 !hba->vreg_info.vccq2) {
7444                 /*
7445                  * Using dev_dbg to avoid messages during runtime PM to avoid
7446                  * never-ending cycles of messages written back to storage by
7447                  * user space causing runtime resume, causing more messages and
7448                  * so on.
7449                  */
7450                 dev_dbg(hba->dev,
7451                         "%s: Regulator capability was not set, actvIccLevel=%d",
7452                                                         __func__, icc_level);
7453                 goto out;
7454         }
7455
7456         if (hba->vreg_info.vcc->max_uA)
7457                 icc_level = ufshcd_get_max_icc_level(
7458                                 hba->vreg_info.vcc->max_uA,
7459                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7460                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7461
7462         if (hba->vreg_info.vccq->max_uA)
7463                 icc_level = ufshcd_get_max_icc_level(
7464                                 hba->vreg_info.vccq->max_uA,
7465                                 icc_level,
7466                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7467
7468         if (hba->vreg_info.vccq2->max_uA)
7469                 icc_level = ufshcd_get_max_icc_level(
7470                                 hba->vreg_info.vccq2->max_uA,
7471                                 icc_level,
7472                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7473 out:
7474         return icc_level;
7475 }
7476
7477 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7478 {
7479         int ret;
7480         int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
7481         u8 *desc_buf;
7482         u32 icc_level;
7483
7484         desc_buf = kmalloc(buff_len, GFP_KERNEL);
7485         if (!desc_buf)
7486                 return;
7487
7488         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7489                                      desc_buf, buff_len);
7490         if (ret) {
7491                 dev_err(hba->dev,
7492                         "%s: Failed reading power descriptor.len = %d ret = %d",
7493                         __func__, buff_len, ret);
7494                 goto out;
7495         }
7496
7497         icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7498                                                          buff_len);
7499         dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7500
7501         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7502                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7503
7504         if (ret)
7505                 dev_err(hba->dev,
7506                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7507                         __func__, icc_level, ret);
7508
7509 out:
7510         kfree(desc_buf);
7511 }
7512
7513 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7514 {
7515         scsi_autopm_get_device(sdev);
7516         blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7517         if (sdev->rpm_autosuspend)
7518                 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7519                                                  RPM_AUTOSUSPEND_DELAY_MS);
7520         scsi_autopm_put_device(sdev);
7521 }
7522
7523 /**
7524  * ufshcd_scsi_add_wlus - Adds required W-LUs
7525  * @hba: per-adapter instance
7526  *
7527  * UFS device specification requires the UFS devices to support 4 well known
7528  * logical units:
7529  *      "REPORT_LUNS" (address: 01h)
7530  *      "UFS Device" (address: 50h)
7531  *      "RPMB" (address: 44h)
7532  *      "BOOT" (address: 30h)
7533  * UFS device's power management needs to be controlled by "POWER CONDITION"
7534  * field of SSU (START STOP UNIT) command. But this "power condition" field
7535  * will take effect only when its sent to "UFS device" well known logical unit
7536  * hence we require the scsi_device instance to represent this logical unit in
7537  * order for the UFS host driver to send the SSU command for power management.
7538  *
7539  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7540  * Block) LU so user space process can control this LU. User space may also
7541  * want to have access to BOOT LU.
7542  *
7543  * This function adds scsi device instances for each of all well known LUs
7544  * (except "REPORT LUNS" LU).
7545  *
7546  * Returns zero on success (all required W-LUs are added successfully),
7547  * non-zero error value on failure (if failed to add any of the required W-LU).
7548  */
7549 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7550 {
7551         int ret = 0;
7552         struct scsi_device *sdev_boot, *sdev_rpmb;
7553
7554         hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
7555                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7556         if (IS_ERR(hba->ufs_device_wlun)) {
7557                 ret = PTR_ERR(hba->ufs_device_wlun);
7558                 hba->ufs_device_wlun = NULL;
7559                 goto out;
7560         }
7561         scsi_device_put(hba->ufs_device_wlun);
7562
7563         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7564                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7565         if (IS_ERR(sdev_rpmb)) {
7566                 ret = PTR_ERR(sdev_rpmb);
7567                 goto remove_ufs_device_wlun;
7568         }
7569         ufshcd_blk_pm_runtime_init(sdev_rpmb);
7570         scsi_device_put(sdev_rpmb);
7571
7572         sdev_boot = __scsi_add_device(hba->host, 0, 0,
7573                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
7574         if (IS_ERR(sdev_boot)) {
7575                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
7576         } else {
7577                 ufshcd_blk_pm_runtime_init(sdev_boot);
7578                 scsi_device_put(sdev_boot);
7579         }
7580         goto out;
7581
7582 remove_ufs_device_wlun:
7583         scsi_remove_device(hba->ufs_device_wlun);
7584 out:
7585         return ret;
7586 }
7587
7588 static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7589 {
7590         struct ufs_dev_info *dev_info = &hba->dev_info;
7591         u8 lun;
7592         u32 d_lu_wb_buf_alloc;
7593         u32 ext_ufs_feature;
7594
7595         if (!ufshcd_is_wb_allowed(hba))
7596                 return;
7597
7598         /*
7599          * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7600          * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7601          * enabled
7602          */
7603         if (!(dev_info->wspecversion >= 0x310 ||
7604               dev_info->wspecversion == 0x220 ||
7605              (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7606                 goto wb_disabled;
7607
7608         if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7609             DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
7610                 goto wb_disabled;
7611
7612         ext_ufs_feature = get_unaligned_be32(desc_buf +
7613                                         DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7614
7615         if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
7616                 goto wb_disabled;
7617
7618         /*
7619          * WB may be supported but not configured while provisioning. The spec
7620          * says, in dedicated wb buffer mode, a max of 1 lun would have wb
7621          * buffer configured.
7622          */
7623         dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7624
7625         dev_info->b_presrv_uspc_en =
7626                 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7627
7628         if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
7629                 if (!get_unaligned_be32(desc_buf +
7630                                    DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
7631                         goto wb_disabled;
7632         } else {
7633                 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7634                         d_lu_wb_buf_alloc = 0;
7635                         ufshcd_read_unit_desc_param(hba,
7636                                         lun,
7637                                         UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7638                                         (u8 *)&d_lu_wb_buf_alloc,
7639                                         sizeof(d_lu_wb_buf_alloc));
7640                         if (d_lu_wb_buf_alloc) {
7641                                 dev_info->wb_dedicated_lu = lun;
7642                                 break;
7643                         }
7644                 }
7645
7646                 if (!d_lu_wb_buf_alloc)
7647                         goto wb_disabled;
7648         }
7649
7650         if (!ufshcd_is_wb_buf_lifetime_available(hba))
7651                 goto wb_disabled;
7652
7653         return;
7654
7655 wb_disabled:
7656         hba->caps &= ~UFSHCD_CAP_WB_EN;
7657 }
7658
7659 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, u8 *desc_buf)
7660 {
7661         struct ufs_dev_info *dev_info = &hba->dev_info;
7662         u32 ext_ufs_feature;
7663         u8 mask = 0;
7664
7665         if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
7666                 return;
7667
7668         ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7669
7670         if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
7671                 mask |= MASK_EE_TOO_LOW_TEMP;
7672
7673         if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
7674                 mask |= MASK_EE_TOO_HIGH_TEMP;
7675
7676         if (mask) {
7677                 ufshcd_enable_ee(hba, mask);
7678                 ufs_hwmon_probe(hba, mask);
7679         }
7680 }
7681
7682 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
7683                              const struct ufs_dev_quirk *fixups)
7684 {
7685         const struct ufs_dev_quirk *f;
7686         struct ufs_dev_info *dev_info = &hba->dev_info;
7687
7688         if (!fixups)
7689                 return;
7690
7691         for (f = fixups; f->quirk; f++) {
7692                 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7693                      f->wmanufacturerid == UFS_ANY_VENDOR) &&
7694                      ((dev_info->model &&
7695                        STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7696                       !strcmp(f->model, UFS_ANY_MODEL)))
7697                         hba->dev_quirks |= f->quirk;
7698         }
7699 }
7700 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
7701
7702 static void ufs_fixup_device_setup(struct ufs_hba *hba)
7703 {
7704         /* fix by general quirk table */
7705         ufshcd_fixup_dev_quirks(hba, ufs_fixups);
7706
7707         /* allow vendors to fix quirks */
7708         ufshcd_vops_fixup_dev_quirks(hba);
7709 }
7710
7711 static int ufs_get_device_desc(struct ufs_hba *hba)
7712 {
7713         int err;
7714         u8 model_index;
7715         u8 b_ufs_feature_sup;
7716         u8 *desc_buf;
7717         struct ufs_dev_info *dev_info = &hba->dev_info;
7718
7719         desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7720         if (!desc_buf) {
7721                 err = -ENOMEM;
7722                 goto out;
7723         }
7724
7725         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7726                                      hba->desc_size[QUERY_DESC_IDN_DEVICE]);
7727         if (err) {
7728                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7729                         __func__, err);
7730                 goto out;
7731         }
7732
7733         /*
7734          * getting vendor (manufacturerID) and Bank Index in big endian
7735          * format
7736          */
7737         dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
7738                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7739
7740         /* getting Specification Version in big endian format */
7741         dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7742                                       desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7743         b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT];
7744
7745         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
7746
7747         if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION &&
7748             (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) {
7749                 bool hpb_en = false;
7750
7751                 ufshpb_get_dev_info(hba, desc_buf);
7752
7753                 if (!ufshpb_is_legacy(hba))
7754                         err = ufshcd_query_flag_retry(hba,
7755                                                       UPIU_QUERY_OPCODE_READ_FLAG,
7756                                                       QUERY_FLAG_IDN_HPB_EN, 0,
7757                                                       &hpb_en);
7758
7759                 if (ufshpb_is_legacy(hba) || (!err && hpb_en))
7760                         dev_info->hpb_enabled = true;
7761         }
7762
7763         err = ufshcd_read_string_desc(hba, model_index,
7764                                       &dev_info->model, SD_ASCII_STD);
7765         if (err < 0) {
7766                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7767                         __func__, err);
7768                 goto out;
7769         }
7770
7771         hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
7772                 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
7773
7774         ufs_fixup_device_setup(hba);
7775
7776         ufshcd_wb_probe(hba, desc_buf);
7777
7778         ufshcd_temp_notif_probe(hba, desc_buf);
7779
7780         /*
7781          * ufshcd_read_string_desc returns size of the string
7782          * reset the error value
7783          */
7784         err = 0;
7785
7786 out:
7787         kfree(desc_buf);
7788         return err;
7789 }
7790
7791 static void ufs_put_device_desc(struct ufs_hba *hba)
7792 {
7793         struct ufs_dev_info *dev_info = &hba->dev_info;
7794
7795         kfree(dev_info->model);
7796         dev_info->model = NULL;
7797 }
7798
7799 /**
7800  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7801  * @hba: per-adapter instance
7802  *
7803  * PA_TActivate parameter can be tuned manually if UniPro version is less than
7804  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7805  * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7806  * the hibern8 exit latency.
7807  *
7808  * Returns zero on success, non-zero error value on failure.
7809  */
7810 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7811 {
7812         int ret = 0;
7813         u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7814
7815         ret = ufshcd_dme_peer_get(hba,
7816                                   UIC_ARG_MIB_SEL(
7817                                         RX_MIN_ACTIVATETIME_CAPABILITY,
7818                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7819                                   &peer_rx_min_activatetime);
7820         if (ret)
7821                 goto out;
7822
7823         /* make sure proper unit conversion is applied */
7824         tuned_pa_tactivate =
7825                 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7826                  / PA_TACTIVATE_TIME_UNIT_US);
7827         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7828                              tuned_pa_tactivate);
7829
7830 out:
7831         return ret;
7832 }
7833
7834 /**
7835  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7836  * @hba: per-adapter instance
7837  *
7838  * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7839  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7840  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7841  * This optimal value can help reduce the hibern8 exit latency.
7842  *
7843  * Returns zero on success, non-zero error value on failure.
7844  */
7845 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7846 {
7847         int ret = 0;
7848         u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7849         u32 max_hibern8_time, tuned_pa_hibern8time;
7850
7851         ret = ufshcd_dme_get(hba,
7852                              UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7853                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7854                                   &local_tx_hibern8_time_cap);
7855         if (ret)
7856                 goto out;
7857
7858         ret = ufshcd_dme_peer_get(hba,
7859                                   UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7860                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7861                                   &peer_rx_hibern8_time_cap);
7862         if (ret)
7863                 goto out;
7864
7865         max_hibern8_time = max(local_tx_hibern8_time_cap,
7866                                peer_rx_hibern8_time_cap);
7867         /* make sure proper unit conversion is applied */
7868         tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7869                                 / PA_HIBERN8_TIME_UNIT_US);
7870         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7871                              tuned_pa_hibern8time);
7872 out:
7873         return ret;
7874 }
7875
7876 /**
7877  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7878  * less than device PA_TACTIVATE time.
7879  * @hba: per-adapter instance
7880  *
7881  * Some UFS devices require host PA_TACTIVATE to be lower than device
7882  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7883  * for such devices.
7884  *
7885  * Returns zero on success, non-zero error value on failure.
7886  */
7887 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7888 {
7889         int ret = 0;
7890         u32 granularity, peer_granularity;
7891         u32 pa_tactivate, peer_pa_tactivate;
7892         u32 pa_tactivate_us, peer_pa_tactivate_us;
7893         u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7894
7895         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7896                                   &granularity);
7897         if (ret)
7898                 goto out;
7899
7900         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7901                                   &peer_granularity);
7902         if (ret)
7903                 goto out;
7904
7905         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7906             (granularity > PA_GRANULARITY_MAX_VAL)) {
7907                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7908                         __func__, granularity);
7909                 return -EINVAL;
7910         }
7911
7912         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7913             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7914                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7915                         __func__, peer_granularity);
7916                 return -EINVAL;
7917         }
7918
7919         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7920         if (ret)
7921                 goto out;
7922
7923         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7924                                   &peer_pa_tactivate);
7925         if (ret)
7926                 goto out;
7927
7928         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7929         peer_pa_tactivate_us = peer_pa_tactivate *
7930                              gran_to_us_table[peer_granularity - 1];
7931
7932         if (pa_tactivate_us >= peer_pa_tactivate_us) {
7933                 u32 new_peer_pa_tactivate;
7934
7935                 new_peer_pa_tactivate = pa_tactivate_us /
7936                                       gran_to_us_table[peer_granularity - 1];
7937                 new_peer_pa_tactivate++;
7938                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7939                                           new_peer_pa_tactivate);
7940         }
7941
7942 out:
7943         return ret;
7944 }
7945
7946 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
7947 {
7948         if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7949                 ufshcd_tune_pa_tactivate(hba);
7950                 ufshcd_tune_pa_hibern8time(hba);
7951         }
7952
7953         ufshcd_vops_apply_dev_quirks(hba);
7954
7955         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7956                 /* set 1ms timeout for PA_TACTIVATE */
7957                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
7958
7959         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7960                 ufshcd_quirk_tune_host_pa_tactivate(hba);
7961 }
7962
7963 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7964 {
7965         hba->ufs_stats.hibern8_exit_cnt = 0;
7966         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7967         hba->req_abort_count = 0;
7968 }
7969
7970 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7971 {
7972         int err;
7973         size_t buff_len;
7974         u8 *desc_buf;
7975
7976         buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
7977         desc_buf = kmalloc(buff_len, GFP_KERNEL);
7978         if (!desc_buf) {
7979                 err = -ENOMEM;
7980                 goto out;
7981         }
7982
7983         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7984                                      desc_buf, buff_len);
7985         if (err) {
7986                 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7987                                 __func__, err);
7988                 goto out;
7989         }
7990
7991         if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7992                 hba->dev_info.max_lu_supported = 32;
7993         else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7994                 hba->dev_info.max_lu_supported = 8;
7995
7996         if (hba->desc_size[QUERY_DESC_IDN_GEOMETRY] >=
7997                 GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS)
7998                 ufshpb_get_geo_info(hba, desc_buf);
7999
8000 out:
8001         kfree(desc_buf);
8002         return err;
8003 }
8004
8005 struct ufs_ref_clk {
8006         unsigned long freq_hz;
8007         enum ufs_ref_clk_freq val;
8008 };
8009
8010 static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8011         {19200000, REF_CLK_FREQ_19_2_MHZ},
8012         {26000000, REF_CLK_FREQ_26_MHZ},
8013         {38400000, REF_CLK_FREQ_38_4_MHZ},
8014         {52000000, REF_CLK_FREQ_52_MHZ},
8015         {0, REF_CLK_FREQ_INVAL},
8016 };
8017
8018 static enum ufs_ref_clk_freq
8019 ufs_get_bref_clk_from_hz(unsigned long freq)
8020 {
8021         int i;
8022
8023         for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8024                 if (ufs_ref_clk_freqs[i].freq_hz == freq)
8025                         return ufs_ref_clk_freqs[i].val;
8026
8027         return REF_CLK_FREQ_INVAL;
8028 }
8029
8030 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8031 {
8032         unsigned long freq;
8033
8034         freq = clk_get_rate(refclk);
8035
8036         hba->dev_ref_clk_freq =
8037                 ufs_get_bref_clk_from_hz(freq);
8038
8039         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8040                 dev_err(hba->dev,
8041                 "invalid ref_clk setting = %ld\n", freq);
8042 }
8043
8044 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8045 {
8046         int err;
8047         u32 ref_clk;
8048         u32 freq = hba->dev_ref_clk_freq;
8049
8050         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8051                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8052
8053         if (err) {
8054                 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8055                         err);
8056                 goto out;
8057         }
8058
8059         if (ref_clk == freq)
8060                 goto out; /* nothing to update */
8061
8062         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8063                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8064
8065         if (err) {
8066                 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8067                         ufs_ref_clk_freqs[freq].freq_hz);
8068                 goto out;
8069         }
8070
8071         dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8072                         ufs_ref_clk_freqs[freq].freq_hz);
8073
8074 out:
8075         return err;
8076 }
8077
8078 static int ufshcd_device_params_init(struct ufs_hba *hba)
8079 {
8080         bool flag;
8081         int ret, i;
8082
8083          /* Init device descriptor sizes */
8084         for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
8085                 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
8086
8087         /* Init UFS geometry descriptor related parameters */
8088         ret = ufshcd_device_geo_params_init(hba);
8089         if (ret)
8090                 goto out;
8091
8092         /* Check and apply UFS device quirks */
8093         ret = ufs_get_device_desc(hba);
8094         if (ret) {
8095                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8096                         __func__, ret);
8097                 goto out;
8098         }
8099
8100         ufshcd_get_ref_clk_gating_wait(hba);
8101
8102         if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8103                         QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8104                 hba->dev_info.f_power_on_wp_en = flag;
8105
8106         /* Probe maximum power mode co-supported by both UFS host and device */
8107         if (ufshcd_get_max_pwr_mode(hba))
8108                 dev_err(hba->dev,
8109                         "%s: Failed getting max supported power mode\n",
8110                         __func__);
8111 out:
8112         return ret;
8113 }
8114
8115 /**
8116  * ufshcd_add_lus - probe and add UFS logical units
8117  * @hba: per-adapter instance
8118  */
8119 static int ufshcd_add_lus(struct ufs_hba *hba)
8120 {
8121         int ret;
8122
8123         /* Add required well known logical units to scsi mid layer */
8124         ret = ufshcd_scsi_add_wlus(hba);
8125         if (ret)
8126                 goto out;
8127
8128         /* Initialize devfreq after UFS device is detected */
8129         if (ufshcd_is_clkscaling_supported(hba)) {
8130                 memcpy(&hba->clk_scaling.saved_pwr_info.info,
8131                         &hba->pwr_info,
8132                         sizeof(struct ufs_pa_layer_attr));
8133                 hba->clk_scaling.saved_pwr_info.is_valid = true;
8134                 hba->clk_scaling.is_allowed = true;
8135
8136                 ret = ufshcd_devfreq_init(hba);
8137                 if (ret)
8138                         goto out;
8139
8140                 hba->clk_scaling.is_enabled = true;
8141                 ufshcd_init_clk_scaling_sysfs(hba);
8142         }
8143
8144         ufs_bsg_probe(hba);
8145         ufshpb_init(hba);
8146         scsi_scan_host(hba->host);
8147         pm_runtime_put_sync(hba->dev);
8148
8149 out:
8150         return ret;
8151 }
8152
8153 /**
8154  * ufshcd_probe_hba - probe hba to detect device and initialize it
8155  * @hba: per-adapter instance
8156  * @init_dev_params: whether or not to call ufshcd_device_params_init().
8157  *
8158  * Execute link-startup and verify device initialization
8159  */
8160 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8161 {
8162         int ret;
8163         unsigned long flags;
8164         ktime_t start = ktime_get();
8165
8166         hba->ufshcd_state = UFSHCD_STATE_RESET;
8167
8168         ret = ufshcd_link_startup(hba);
8169         if (ret)
8170                 goto out;
8171
8172         if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8173                 goto out;
8174
8175         /* Debug counters initialization */
8176         ufshcd_clear_dbg_ufs_stats(hba);
8177
8178         /* UniPro link is active now */
8179         ufshcd_set_link_active(hba);
8180
8181         /* Verify device initialization by sending NOP OUT UPIU */
8182         ret = ufshcd_verify_dev_init(hba);
8183         if (ret)
8184                 goto out;
8185
8186         /* Initiate UFS initialization, and waiting until completion */
8187         ret = ufshcd_complete_dev_init(hba);
8188         if (ret)
8189                 goto out;
8190
8191         /*
8192          * Initialize UFS device parameters used by driver, these
8193          * parameters are associated with UFS descriptors.
8194          */
8195         if (init_dev_params) {
8196                 ret = ufshcd_device_params_init(hba);
8197                 if (ret)
8198                         goto out;
8199         }
8200
8201         ufshcd_tune_unipro_params(hba);
8202
8203         /* UFS device is also active now */
8204         ufshcd_set_ufs_dev_active(hba);
8205         ufshcd_force_reset_auto_bkops(hba);
8206
8207         /* Gear up to HS gear if supported */
8208         if (hba->max_pwr_info.is_valid) {
8209                 /*
8210                  * Set the right value to bRefClkFreq before attempting to
8211                  * switch to HS gears.
8212                  */
8213                 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8214                         ufshcd_set_dev_ref_clk(hba);
8215                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8216                 if (ret) {
8217                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8218                                         __func__, ret);
8219                         goto out;
8220                 }
8221                 ufshcd_print_pwr_info(hba);
8222         }
8223
8224         /*
8225          * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8226          * and for removable UFS card as well, hence always set the parameter.
8227          * Note: Error handler may issue the device reset hence resetting
8228          * bActiveICCLevel as well so it is always safe to set this here.
8229          */
8230         ufshcd_set_active_icc_lvl(hba);
8231
8232         ufshcd_wb_config(hba);
8233         if (hba->ee_usr_mask)
8234                 ufshcd_write_ee_control(hba);
8235         /* Enable Auto-Hibernate if configured */
8236         ufshcd_auto_hibern8_enable(hba);
8237
8238         ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT);
8239 out:
8240         spin_lock_irqsave(hba->host->host_lock, flags);
8241         if (ret)
8242                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8243         else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8244                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8245         spin_unlock_irqrestore(hba->host->host_lock, flags);
8246
8247         trace_ufshcd_init(dev_name(hba->dev), ret,
8248                 ktime_to_us(ktime_sub(ktime_get(), start)),
8249                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8250         return ret;
8251 }
8252
8253 /**
8254  * ufshcd_async_scan - asynchronous execution for probing hba
8255  * @data: data pointer to pass to this function
8256  * @cookie: cookie data
8257  */
8258 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8259 {
8260         struct ufs_hba *hba = (struct ufs_hba *)data;
8261         int ret;
8262
8263         down(&hba->host_sem);
8264         /* Initialize hba, detect and initialize UFS device */
8265         ret = ufshcd_probe_hba(hba, true);
8266         up(&hba->host_sem);
8267         if (ret)
8268                 goto out;
8269
8270         /* Probe and add UFS logical units  */
8271         ret = ufshcd_add_lus(hba);
8272 out:
8273         /*
8274          * If we failed to initialize the device or the device is not
8275          * present, turn off the power/clocks etc.
8276          */
8277         if (ret) {
8278                 pm_runtime_put_sync(hba->dev);
8279                 ufshcd_hba_exit(hba);
8280         }
8281 }
8282
8283 static const struct attribute_group *ufshcd_driver_groups[] = {
8284         &ufs_sysfs_unit_descriptor_group,
8285         &ufs_sysfs_lun_attributes_group,
8286 #ifdef CONFIG_SCSI_UFS_HPB
8287         &ufs_sysfs_hpb_stat_group,
8288         &ufs_sysfs_hpb_param_group,
8289 #endif
8290         NULL,
8291 };
8292
8293 static struct ufs_hba_variant_params ufs_hba_vps = {
8294         .hba_enable_delay_us            = 1000,
8295         .wb_flush_threshold             = UFS_WB_BUF_REMAIN_PERCENT(40),
8296         .devfreq_profile.polling_ms     = 100,
8297         .devfreq_profile.target         = ufshcd_devfreq_target,
8298         .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8299         .ondemand_data.upthreshold      = 70,
8300         .ondemand_data.downdifferential = 5,
8301 };
8302
8303 static struct scsi_host_template ufshcd_driver_template = {
8304         .module                 = THIS_MODULE,
8305         .name                   = UFSHCD,
8306         .proc_name              = UFSHCD,
8307         .map_queues             = ufshcd_map_queues,
8308         .queuecommand           = ufshcd_queuecommand,
8309         .mq_poll                = ufshcd_poll,
8310         .slave_alloc            = ufshcd_slave_alloc,
8311         .slave_configure        = ufshcd_slave_configure,
8312         .slave_destroy          = ufshcd_slave_destroy,
8313         .change_queue_depth     = ufshcd_change_queue_depth,
8314         .eh_abort_handler       = ufshcd_abort,
8315         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8316         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
8317         .this_id                = -1,
8318         .sg_tablesize           = SG_ALL,
8319         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
8320         .can_queue              = UFSHCD_CAN_QUEUE,
8321         .max_segment_size       = PRDT_DATA_BYTE_COUNT_MAX,
8322         .max_host_blocked       = 1,
8323         .track_queue_depth      = 1,
8324         .sdev_groups            = ufshcd_driver_groups,
8325         .dma_boundary           = PAGE_SIZE - 1,
8326         .rpm_autosuspend_delay  = RPM_AUTOSUSPEND_DELAY_MS,
8327 };
8328
8329 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8330                                    int ua)
8331 {
8332         int ret;
8333
8334         if (!vreg)
8335                 return 0;
8336
8337         /*
8338          * "set_load" operation shall be required on those regulators
8339          * which specifically configured current limitation. Otherwise
8340          * zero max_uA may cause unexpected behavior when regulator is
8341          * enabled or set as high power mode.
8342          */
8343         if (!vreg->max_uA)
8344                 return 0;
8345
8346         ret = regulator_set_load(vreg->reg, ua);
8347         if (ret < 0) {
8348                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8349                                 __func__, vreg->name, ua, ret);
8350         }
8351
8352         return ret;
8353 }
8354
8355 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8356                                          struct ufs_vreg *vreg)
8357 {
8358         return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
8359 }
8360
8361 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8362                                          struct ufs_vreg *vreg)
8363 {
8364         if (!vreg)
8365                 return 0;
8366
8367         return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
8368 }
8369
8370 static int ufshcd_config_vreg(struct device *dev,
8371                 struct ufs_vreg *vreg, bool on)
8372 {
8373         if (regulator_count_voltages(vreg->reg) <= 0)
8374                 return 0;
8375
8376         return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
8377 }
8378
8379 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8380 {
8381         int ret = 0;
8382
8383         if (!vreg || vreg->enabled)
8384                 goto out;
8385
8386         ret = ufshcd_config_vreg(dev, vreg, true);
8387         if (!ret)
8388                 ret = regulator_enable(vreg->reg);
8389
8390         if (!ret)
8391                 vreg->enabled = true;
8392         else
8393                 dev_err(dev, "%s: %s enable failed, err=%d\n",
8394                                 __func__, vreg->name, ret);
8395 out:
8396         return ret;
8397 }
8398
8399 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8400 {
8401         int ret = 0;
8402
8403         if (!vreg || !vreg->enabled || vreg->always_on)
8404                 goto out;
8405
8406         ret = regulator_disable(vreg->reg);
8407
8408         if (!ret) {
8409                 /* ignore errors on applying disable config */
8410                 ufshcd_config_vreg(dev, vreg, false);
8411                 vreg->enabled = false;
8412         } else {
8413                 dev_err(dev, "%s: %s disable failed, err=%d\n",
8414                                 __func__, vreg->name, ret);
8415         }
8416 out:
8417         return ret;
8418 }
8419
8420 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8421 {
8422         int ret = 0;
8423         struct device *dev = hba->dev;
8424         struct ufs_vreg_info *info = &hba->vreg_info;
8425
8426         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8427         if (ret)
8428                 goto out;
8429
8430         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8431         if (ret)
8432                 goto out;
8433
8434         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
8435
8436 out:
8437         if (ret) {
8438                 ufshcd_toggle_vreg(dev, info->vccq2, false);
8439                 ufshcd_toggle_vreg(dev, info->vccq, false);
8440                 ufshcd_toggle_vreg(dev, info->vcc, false);
8441         }
8442         return ret;
8443 }
8444
8445 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8446 {
8447         struct ufs_vreg_info *info = &hba->vreg_info;
8448
8449         return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
8450 }
8451
8452 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8453 {
8454         int ret = 0;
8455
8456         if (!vreg)
8457                 goto out;
8458
8459         vreg->reg = devm_regulator_get(dev, vreg->name);
8460         if (IS_ERR(vreg->reg)) {
8461                 ret = PTR_ERR(vreg->reg);
8462                 dev_err(dev, "%s: %s get failed, err=%d\n",
8463                                 __func__, vreg->name, ret);
8464         }
8465 out:
8466         return ret;
8467 }
8468
8469 static int ufshcd_init_vreg(struct ufs_hba *hba)
8470 {
8471         int ret = 0;
8472         struct device *dev = hba->dev;
8473         struct ufs_vreg_info *info = &hba->vreg_info;
8474
8475         ret = ufshcd_get_vreg(dev, info->vcc);
8476         if (ret)
8477                 goto out;
8478
8479         ret = ufshcd_get_vreg(dev, info->vccq);
8480         if (!ret)
8481                 ret = ufshcd_get_vreg(dev, info->vccq2);
8482 out:
8483         return ret;
8484 }
8485
8486 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8487 {
8488         struct ufs_vreg_info *info = &hba->vreg_info;
8489
8490         return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8491 }
8492
8493 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8494 {
8495         int ret = 0;
8496         struct ufs_clk_info *clki;
8497         struct list_head *head = &hba->clk_list_head;
8498         unsigned long flags;
8499         ktime_t start = ktime_get();
8500         bool clk_state_changed = false;
8501
8502         if (list_empty(head))
8503                 goto out;
8504
8505         ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8506         if (ret)
8507                 return ret;
8508
8509         list_for_each_entry(clki, head, list) {
8510                 if (!IS_ERR_OR_NULL(clki->clk)) {
8511                         /*
8512                          * Don't disable clocks which are needed
8513                          * to keep the link active.
8514                          */
8515                         if (ufshcd_is_link_active(hba) &&
8516                             clki->keep_link_active)
8517                                 continue;
8518
8519                         clk_state_changed = on ^ clki->enabled;
8520                         if (on && !clki->enabled) {
8521                                 ret = clk_prepare_enable(clki->clk);
8522                                 if (ret) {
8523                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8524                                                 __func__, clki->name, ret);
8525                                         goto out;
8526                                 }
8527                         } else if (!on && clki->enabled) {
8528                                 clk_disable_unprepare(clki->clk);
8529                         }
8530                         clki->enabled = on;
8531                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8532                                         clki->name, on ? "en" : "dis");
8533                 }
8534         }
8535
8536         ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8537         if (ret)
8538                 return ret;
8539
8540 out:
8541         if (ret) {
8542                 list_for_each_entry(clki, head, list) {
8543                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8544                                 clk_disable_unprepare(clki->clk);
8545                 }
8546         } else if (!ret && on) {
8547                 spin_lock_irqsave(hba->host->host_lock, flags);
8548                 hba->clk_gating.state = CLKS_ON;
8549                 trace_ufshcd_clk_gating(dev_name(hba->dev),
8550                                         hba->clk_gating.state);
8551                 spin_unlock_irqrestore(hba->host->host_lock, flags);
8552         }
8553
8554         if (clk_state_changed)
8555                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8556                         (on ? "on" : "off"),
8557                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
8558         return ret;
8559 }
8560
8561 static int ufshcd_init_clocks(struct ufs_hba *hba)
8562 {
8563         int ret = 0;
8564         struct ufs_clk_info *clki;
8565         struct device *dev = hba->dev;
8566         struct list_head *head = &hba->clk_list_head;
8567
8568         if (list_empty(head))
8569                 goto out;
8570
8571         list_for_each_entry(clki, head, list) {
8572                 if (!clki->name)
8573                         continue;
8574
8575                 clki->clk = devm_clk_get(dev, clki->name);
8576                 if (IS_ERR(clki->clk)) {
8577                         ret = PTR_ERR(clki->clk);
8578                         dev_err(dev, "%s: %s clk get failed, %d\n",
8579                                         __func__, clki->name, ret);
8580                         goto out;
8581                 }
8582
8583                 /*
8584                  * Parse device ref clk freq as per device tree "ref_clk".
8585                  * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8586                  * in ufshcd_alloc_host().
8587                  */
8588                 if (!strcmp(clki->name, "ref_clk"))
8589                         ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8590
8591                 if (clki->max_freq) {
8592                         ret = clk_set_rate(clki->clk, clki->max_freq);
8593                         if (ret) {
8594                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8595                                         __func__, clki->name,
8596                                         clki->max_freq, ret);
8597                                 goto out;
8598                         }
8599                         clki->curr_freq = clki->max_freq;
8600                 }
8601                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8602                                 clki->name, clk_get_rate(clki->clk));
8603         }
8604 out:
8605         return ret;
8606 }
8607
8608 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8609 {
8610         int err = 0;
8611
8612         if (!hba->vops)
8613                 goto out;
8614
8615         err = ufshcd_vops_init(hba);
8616         if (err)
8617                 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
8618                         __func__, ufshcd_get_var_name(hba), err);
8619 out:
8620         return err;
8621 }
8622
8623 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8624 {
8625         if (!hba->vops)
8626                 return;
8627
8628         ufshcd_vops_exit(hba);
8629 }
8630
8631 static int ufshcd_hba_init(struct ufs_hba *hba)
8632 {
8633         int err;
8634
8635         /*
8636          * Handle host controller power separately from the UFS device power
8637          * rails as it will help controlling the UFS host controller power
8638          * collapse easily which is different than UFS device power collapse.
8639          * Also, enable the host controller power before we go ahead with rest
8640          * of the initialization here.
8641          */
8642         err = ufshcd_init_hba_vreg(hba);
8643         if (err)
8644                 goto out;
8645
8646         err = ufshcd_setup_hba_vreg(hba, true);
8647         if (err)
8648                 goto out;
8649
8650         err = ufshcd_init_clocks(hba);
8651         if (err)
8652                 goto out_disable_hba_vreg;
8653
8654         err = ufshcd_setup_clocks(hba, true);
8655         if (err)
8656                 goto out_disable_hba_vreg;
8657
8658         err = ufshcd_init_vreg(hba);
8659         if (err)
8660                 goto out_disable_clks;
8661
8662         err = ufshcd_setup_vreg(hba, true);
8663         if (err)
8664                 goto out_disable_clks;
8665
8666         err = ufshcd_variant_hba_init(hba);
8667         if (err)
8668                 goto out_disable_vreg;
8669
8670         ufs_debugfs_hba_init(hba);
8671
8672         hba->is_powered = true;
8673         goto out;
8674
8675 out_disable_vreg:
8676         ufshcd_setup_vreg(hba, false);
8677 out_disable_clks:
8678         ufshcd_setup_clocks(hba, false);
8679 out_disable_hba_vreg:
8680         ufshcd_setup_hba_vreg(hba, false);
8681 out:
8682         return err;
8683 }
8684
8685 static void ufshcd_hba_exit(struct ufs_hba *hba)
8686 {
8687         if (hba->is_powered) {
8688                 ufshcd_exit_clk_scaling(hba);
8689                 ufshcd_exit_clk_gating(hba);
8690                 if (hba->eh_wq)
8691                         destroy_workqueue(hba->eh_wq);
8692                 ufs_debugfs_hba_exit(hba);
8693                 ufshcd_variant_hba_exit(hba);
8694                 ufshcd_setup_vreg(hba, false);
8695                 ufshcd_setup_clocks(hba, false);
8696                 ufshcd_setup_hba_vreg(hba, false);
8697                 hba->is_powered = false;
8698                 ufs_put_device_desc(hba);
8699         }
8700 }
8701
8702 /**
8703  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8704  *                           power mode
8705  * @hba: per adapter instance
8706  * @pwr_mode: device power mode to set
8707  *
8708  * Returns 0 if requested power mode is set successfully
8709  * Returns < 0 if failed to set the requested power mode
8710  */
8711 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8712                                      enum ufs_dev_pwr_mode pwr_mode)
8713 {
8714         unsigned char cmd[6] = { START_STOP };
8715         struct scsi_sense_hdr sshdr;
8716         struct scsi_device *sdp;
8717         unsigned long flags;
8718         int ret, retries;
8719
8720         spin_lock_irqsave(hba->host->host_lock, flags);
8721         sdp = hba->ufs_device_wlun;
8722         if (sdp) {
8723                 ret = scsi_device_get(sdp);
8724                 if (!ret && !scsi_device_online(sdp)) {
8725                         ret = -ENODEV;
8726                         scsi_device_put(sdp);
8727                 }
8728         } else {
8729                 ret = -ENODEV;
8730         }
8731         spin_unlock_irqrestore(hba->host->host_lock, flags);
8732
8733         if (ret)
8734                 return ret;
8735
8736         /*
8737          * If scsi commands fail, the scsi mid-layer schedules scsi error-
8738          * handling, which would wait for host to be resumed. Since we know
8739          * we are functional while we are here, skip host resume in error
8740          * handling context.
8741          */
8742         hba->host->eh_noresume = 1;
8743
8744         cmd[4] = pwr_mode << 4;
8745
8746         /*
8747          * Current function would be generally called from the power management
8748          * callbacks hence set the RQF_PM flag so that it doesn't resume the
8749          * already suspended childs.
8750          */
8751         for (retries = 3; retries > 0; --retries) {
8752                 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8753                                 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
8754                 if (!scsi_status_is_check_condition(ret) ||
8755                                 !scsi_sense_valid(&sshdr) ||
8756                                 sshdr.sense_key != UNIT_ATTENTION)
8757                         break;
8758         }
8759         if (ret) {
8760                 sdev_printk(KERN_WARNING, sdp,
8761                             "START_STOP failed for power mode: %d, result %x\n",
8762                             pwr_mode, ret);
8763                 if (ret > 0) {
8764                         if (scsi_sense_valid(&sshdr))
8765                                 scsi_print_sense_hdr(sdp, NULL, &sshdr);
8766                         ret = -EIO;
8767                 }
8768         }
8769
8770         if (!ret)
8771                 hba->curr_dev_pwr_mode = pwr_mode;
8772
8773         scsi_device_put(sdp);
8774         hba->host->eh_noresume = 0;
8775         return ret;
8776 }
8777
8778 static int ufshcd_link_state_transition(struct ufs_hba *hba,
8779                                         enum uic_link_state req_link_state,
8780                                         int check_for_bkops)
8781 {
8782         int ret = 0;
8783
8784         if (req_link_state == hba->uic_link_state)
8785                 return 0;
8786
8787         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8788                 ret = ufshcd_uic_hibern8_enter(hba);
8789                 if (!ret) {
8790                         ufshcd_set_link_hibern8(hba);
8791                 } else {
8792                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8793                                         __func__, ret);
8794                         goto out;
8795                 }
8796         }
8797         /*
8798          * If autobkops is enabled, link can't be turned off because
8799          * turning off the link would also turn off the device, except in the
8800          * case of DeepSleep where the device is expected to remain powered.
8801          */
8802         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
8803                  (!check_for_bkops || !hba->auto_bkops_enabled)) {
8804                 /*
8805                  * Let's make sure that link is in low power mode, we are doing
8806                  * this currently by putting the link in Hibern8. Otherway to
8807                  * put the link in low power mode is to send the DME end point
8808                  * to device and then send the DME reset command to local
8809                  * unipro. But putting the link in hibern8 is much faster.
8810                  *
8811                  * Note also that putting the link in Hibern8 is a requirement
8812                  * for entering DeepSleep.
8813                  */
8814                 ret = ufshcd_uic_hibern8_enter(hba);
8815                 if (ret) {
8816                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8817                                         __func__, ret);
8818                         goto out;
8819                 }
8820                 /*
8821                  * Change controller state to "reset state" which
8822                  * should also put the link in off/reset state
8823                  */
8824                 ufshcd_hba_stop(hba);
8825                 /*
8826                  * TODO: Check if we need any delay to make sure that
8827                  * controller is reset
8828                  */
8829                 ufshcd_set_link_off(hba);
8830         }
8831
8832 out:
8833         return ret;
8834 }
8835
8836 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8837 {
8838         bool vcc_off = false;
8839
8840         /*
8841          * It seems some UFS devices may keep drawing more than sleep current
8842          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8843          * To avoid this situation, add 2ms delay before putting these UFS
8844          * rails in LPM mode.
8845          */
8846         if (!ufshcd_is_link_active(hba) &&
8847             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8848                 usleep_range(2000, 2100);
8849
8850         /*
8851          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8852          * power.
8853          *
8854          * If UFS device and link is in OFF state, all power supplies (VCC,
8855          * VCCQ, VCCQ2) can be turned off if power on write protect is not
8856          * required. If UFS link is inactive (Hibern8 or OFF state) and device
8857          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8858          *
8859          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8860          * in low power state which would save some power.
8861          *
8862          * If Write Booster is enabled and the device needs to flush the WB
8863          * buffer OR if bkops status is urgent for WB, keep Vcc on.
8864          */
8865         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8866             !hba->dev_info.is_lu_power_on_wp) {
8867                 ufshcd_setup_vreg(hba, false);
8868                 vcc_off = true;
8869         } else if (!ufshcd_is_ufs_dev_active(hba)) {
8870                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8871                 vcc_off = true;
8872                 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
8873                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8874                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8875                 }
8876         }
8877
8878         /*
8879          * Some UFS devices require delay after VCC power rail is turned-off.
8880          */
8881         if (vcc_off && hba->vreg_info.vcc &&
8882                 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8883                 usleep_range(5000, 5100);
8884 }
8885
8886 #ifdef CONFIG_PM
8887 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8888 {
8889         int ret = 0;
8890
8891         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8892             !hba->dev_info.is_lu_power_on_wp) {
8893                 ret = ufshcd_setup_vreg(hba, true);
8894         } else if (!ufshcd_is_ufs_dev_active(hba)) {
8895                 if (!ufshcd_is_link_active(hba)) {
8896                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8897                         if (ret)
8898                                 goto vcc_disable;
8899                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8900                         if (ret)
8901                                 goto vccq_lpm;
8902                 }
8903                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
8904         }
8905         goto out;
8906
8907 vccq_lpm:
8908         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8909 vcc_disable:
8910         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8911 out:
8912         return ret;
8913 }
8914 #endif /* CONFIG_PM */
8915
8916 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8917 {
8918         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8919                 ufshcd_setup_hba_vreg(hba, false);
8920 }
8921
8922 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8923 {
8924         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8925                 ufshcd_setup_hba_vreg(hba, true);
8926 }
8927
8928 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8929 {
8930         int ret = 0;
8931         int check_for_bkops;
8932         enum ufs_pm_level pm_lvl;
8933         enum ufs_dev_pwr_mode req_dev_pwr_mode;
8934         enum uic_link_state req_link_state;
8935
8936         hba->pm_op_in_progress = true;
8937         if (pm_op != UFS_SHUTDOWN_PM) {
8938                 pm_lvl = pm_op == UFS_RUNTIME_PM ?
8939                          hba->rpm_lvl : hba->spm_lvl;
8940                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8941                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8942         } else {
8943                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8944                 req_link_state = UIC_LINK_OFF_STATE;
8945         }
8946
8947         ufshpb_suspend(hba);
8948
8949         /*
8950          * If we can't transition into any of the low power modes
8951          * just gate the clocks.
8952          */
8953         ufshcd_hold(hba, false);
8954         hba->clk_gating.is_suspended = true;
8955
8956         if (ufshcd_is_clkscaling_supported(hba))
8957                 ufshcd_clk_scaling_suspend(hba, true);
8958
8959         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8960                         req_link_state == UIC_LINK_ACTIVE_STATE) {
8961                 goto vops_suspend;
8962         }
8963
8964         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8965             (req_link_state == hba->uic_link_state))
8966                 goto enable_scaling;
8967
8968         /* UFS device & link must be active before we enter in this function */
8969         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8970                 ret = -EINVAL;
8971                 goto enable_scaling;
8972         }
8973
8974         if (pm_op == UFS_RUNTIME_PM) {
8975                 if (ufshcd_can_autobkops_during_suspend(hba)) {
8976                         /*
8977                          * The device is idle with no requests in the queue,
8978                          * allow background operations if bkops status shows
8979                          * that performance might be impacted.
8980                          */
8981                         ret = ufshcd_urgent_bkops(hba);
8982                         if (ret)
8983                                 goto enable_scaling;
8984                 } else {
8985                         /* make sure that auto bkops is disabled */
8986                         ufshcd_disable_auto_bkops(hba);
8987                 }
8988                 /*
8989                  * If device needs to do BKOP or WB buffer flush during
8990                  * Hibern8, keep device power mode as "active power mode"
8991                  * and VCC supply.
8992                  */
8993                 hba->dev_info.b_rpm_dev_flush_capable =
8994                         hba->auto_bkops_enabled ||
8995                         (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8996                         ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8997                         ufshcd_is_auto_hibern8_enabled(hba))) &&
8998                         ufshcd_wb_need_flush(hba));
8999         }
9000
9001         flush_work(&hba->eeh_work);
9002
9003         ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9004         if (ret)
9005                 goto enable_scaling;
9006
9007         if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9008                 if (pm_op != UFS_RUNTIME_PM)
9009                         /* ensure that bkops is disabled */
9010                         ufshcd_disable_auto_bkops(hba);
9011
9012                 if (!hba->dev_info.b_rpm_dev_flush_capable) {
9013                         ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9014                         if (ret)
9015                                 goto enable_scaling;
9016                 }
9017         }
9018
9019         /*
9020          * In the case of DeepSleep, the device is expected to remain powered
9021          * with the link off, so do not check for bkops.
9022          */
9023         check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9024         ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9025         if (ret)
9026                 goto set_dev_active;
9027
9028 vops_suspend:
9029         /*
9030          * Call vendor specific suspend callback. As these callbacks may access
9031          * vendor specific host controller register space call them before the
9032          * host clocks are ON.
9033          */
9034         ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9035         if (ret)
9036                 goto set_link_active;
9037         goto out;
9038
9039 set_link_active:
9040         /*
9041          * Device hardware reset is required to exit DeepSleep. Also, for
9042          * DeepSleep, the link is off so host reset and restore will be done
9043          * further below.
9044          */
9045         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9046                 ufshcd_device_reset(hba);
9047                 WARN_ON(!ufshcd_is_link_off(hba));
9048         }
9049         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9050                 ufshcd_set_link_active(hba);
9051         else if (ufshcd_is_link_off(hba))
9052                 ufshcd_host_reset_and_restore(hba);
9053 set_dev_active:
9054         /* Can also get here needing to exit DeepSleep */
9055         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9056                 ufshcd_device_reset(hba);
9057                 ufshcd_host_reset_and_restore(hba);
9058         }
9059         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9060                 ufshcd_disable_auto_bkops(hba);
9061 enable_scaling:
9062         if (ufshcd_is_clkscaling_supported(hba))
9063                 ufshcd_clk_scaling_suspend(hba, false);
9064
9065         hba->dev_info.b_rpm_dev_flush_capable = false;
9066 out:
9067         if (hba->dev_info.b_rpm_dev_flush_capable) {
9068                 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9069                         msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9070         }
9071
9072         if (ret) {
9073                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9074                 hba->clk_gating.is_suspended = false;
9075                 ufshcd_release(hba);
9076                 ufshpb_resume(hba);
9077         }
9078         hba->pm_op_in_progress = false;
9079         return ret;
9080 }
9081
9082 #ifdef CONFIG_PM
9083 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9084 {
9085         int ret;
9086         enum uic_link_state old_link_state = hba->uic_link_state;
9087
9088         hba->pm_op_in_progress = true;
9089
9090         /*
9091          * Call vendor specific resume callback. As these callbacks may access
9092          * vendor specific host controller register space call them when the
9093          * host clocks are ON.
9094          */
9095         ret = ufshcd_vops_resume(hba, pm_op);
9096         if (ret)
9097                 goto out;
9098
9099         /* For DeepSleep, the only supported option is to have the link off */
9100         WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9101
9102         if (ufshcd_is_link_hibern8(hba)) {
9103                 ret = ufshcd_uic_hibern8_exit(hba);
9104                 if (!ret) {
9105                         ufshcd_set_link_active(hba);
9106                 } else {
9107                         dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9108                                         __func__, ret);
9109                         goto vendor_suspend;
9110                 }
9111         } else if (ufshcd_is_link_off(hba)) {
9112                 /*
9113                  * A full initialization of the host and the device is
9114                  * required since the link was put to off during suspend.
9115                  * Note, in the case of DeepSleep, the device will exit
9116                  * DeepSleep due to device reset.
9117                  */
9118                 ret = ufshcd_reset_and_restore(hba);
9119                 /*
9120                  * ufshcd_reset_and_restore() should have already
9121                  * set the link state as active
9122                  */
9123                 if (ret || !ufshcd_is_link_active(hba))
9124                         goto vendor_suspend;
9125         }
9126
9127         if (!ufshcd_is_ufs_dev_active(hba)) {
9128                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9129                 if (ret)
9130                         goto set_old_link_state;
9131         }
9132
9133         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9134                 ufshcd_enable_auto_bkops(hba);
9135         else
9136                 /*
9137                  * If BKOPs operations are urgently needed at this moment then
9138                  * keep auto-bkops enabled or else disable it.
9139                  */
9140                 ufshcd_urgent_bkops(hba);
9141
9142         if (hba->ee_usr_mask)
9143                 ufshcd_write_ee_control(hba);
9144
9145         if (ufshcd_is_clkscaling_supported(hba))
9146                 ufshcd_clk_scaling_suspend(hba, false);
9147
9148         if (hba->dev_info.b_rpm_dev_flush_capable) {
9149                 hba->dev_info.b_rpm_dev_flush_capable = false;
9150                 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9151         }
9152
9153         /* Enable Auto-Hibernate if configured */
9154         ufshcd_auto_hibern8_enable(hba);
9155
9156         ufshpb_resume(hba);
9157         goto out;
9158
9159 set_old_link_state:
9160         ufshcd_link_state_transition(hba, old_link_state, 0);
9161 vendor_suspend:
9162         ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9163         ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9164 out:
9165         if (ret)
9166                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9167         hba->clk_gating.is_suspended = false;
9168         ufshcd_release(hba);
9169         hba->pm_op_in_progress = false;
9170         return ret;
9171 }
9172
9173 static int ufshcd_wl_runtime_suspend(struct device *dev)
9174 {
9175         struct scsi_device *sdev = to_scsi_device(dev);
9176         struct ufs_hba *hba;
9177         int ret;
9178         ktime_t start = ktime_get();
9179
9180         hba = shost_priv(sdev->host);
9181
9182         ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9183         if (ret)
9184                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9185
9186         trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9187                 ktime_to_us(ktime_sub(ktime_get(), start)),
9188                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9189
9190         return ret;
9191 }
9192
9193 static int ufshcd_wl_runtime_resume(struct device *dev)
9194 {
9195         struct scsi_device *sdev = to_scsi_device(dev);
9196         struct ufs_hba *hba;
9197         int ret = 0;
9198         ktime_t start = ktime_get();
9199
9200         hba = shost_priv(sdev->host);
9201
9202         ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9203         if (ret)
9204                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9205
9206         trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9207                 ktime_to_us(ktime_sub(ktime_get(), start)),
9208                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9209
9210         return ret;
9211 }
9212 #endif
9213
9214 #ifdef CONFIG_PM_SLEEP
9215 static int ufshcd_wl_suspend(struct device *dev)
9216 {
9217         struct scsi_device *sdev = to_scsi_device(dev);
9218         struct ufs_hba *hba;
9219         int ret = 0;
9220         ktime_t start = ktime_get();
9221
9222         hba = shost_priv(sdev->host);
9223         down(&hba->host_sem);
9224
9225         if (pm_runtime_suspended(dev))
9226                 goto out;
9227
9228         ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9229         if (ret) {
9230                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
9231                 up(&hba->host_sem);
9232         }
9233
9234 out:
9235         if (!ret)
9236                 hba->is_sys_suspended = true;
9237         trace_ufshcd_wl_suspend(dev_name(dev), ret,
9238                 ktime_to_us(ktime_sub(ktime_get(), start)),
9239                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9240
9241         return ret;
9242 }
9243
9244 static int ufshcd_wl_resume(struct device *dev)
9245 {
9246         struct scsi_device *sdev = to_scsi_device(dev);
9247         struct ufs_hba *hba;
9248         int ret = 0;
9249         ktime_t start = ktime_get();
9250
9251         hba = shost_priv(sdev->host);
9252
9253         if (pm_runtime_suspended(dev))
9254                 goto out;
9255
9256         ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9257         if (ret)
9258                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9259 out:
9260         trace_ufshcd_wl_resume(dev_name(dev), ret,
9261                 ktime_to_us(ktime_sub(ktime_get(), start)),
9262                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9263         if (!ret)
9264                 hba->is_sys_suspended = false;
9265         up(&hba->host_sem);
9266         return ret;
9267 }
9268 #endif
9269
9270 static void ufshcd_wl_shutdown(struct device *dev)
9271 {
9272         struct scsi_device *sdev = to_scsi_device(dev);
9273         struct ufs_hba *hba;
9274
9275         hba = shost_priv(sdev->host);
9276
9277         down(&hba->host_sem);
9278         hba->shutting_down = true;
9279         up(&hba->host_sem);
9280
9281         /* Turn on everything while shutting down */
9282         ufshcd_rpm_get_sync(hba);
9283         scsi_device_quiesce(sdev);
9284         shost_for_each_device(sdev, hba->host) {
9285                 if (sdev == hba->ufs_device_wlun)
9286                         continue;
9287                 scsi_device_quiesce(sdev);
9288         }
9289         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9290 }
9291
9292 /**
9293  * ufshcd_suspend - helper function for suspend operations
9294  * @hba: per adapter instance
9295  *
9296  * This function will put disable irqs, turn off clocks
9297  * and set vreg and hba-vreg in lpm mode.
9298  */
9299 static int ufshcd_suspend(struct ufs_hba *hba)
9300 {
9301         int ret;
9302
9303         if (!hba->is_powered)
9304                 return 0;
9305         /*
9306          * Disable the host irq as host controller as there won't be any
9307          * host controller transaction expected till resume.
9308          */
9309         ufshcd_disable_irq(hba);
9310         ret = ufshcd_setup_clocks(hba, false);
9311         if (ret) {
9312                 ufshcd_enable_irq(hba);
9313                 return ret;
9314         }
9315         if (ufshcd_is_clkgating_allowed(hba)) {
9316                 hba->clk_gating.state = CLKS_OFF;
9317                 trace_ufshcd_clk_gating(dev_name(hba->dev),
9318                                         hba->clk_gating.state);
9319         }
9320
9321         ufshcd_vreg_set_lpm(hba);
9322         /* Put the host controller in low power mode if possible */
9323         ufshcd_hba_vreg_set_lpm(hba);
9324         return ret;
9325 }
9326
9327 #ifdef CONFIG_PM
9328 /**
9329  * ufshcd_resume - helper function for resume operations
9330  * @hba: per adapter instance
9331  *
9332  * This function basically turns on the regulators, clocks and
9333  * irqs of the hba.
9334  *
9335  * Returns 0 for success and non-zero for failure
9336  */
9337 static int ufshcd_resume(struct ufs_hba *hba)
9338 {
9339         int ret;
9340
9341         if (!hba->is_powered)
9342                 return 0;
9343
9344         ufshcd_hba_vreg_set_hpm(hba);
9345         ret = ufshcd_vreg_set_hpm(hba);
9346         if (ret)
9347                 goto out;
9348
9349         /* Make sure clocks are enabled before accessing controller */
9350         ret = ufshcd_setup_clocks(hba, true);
9351         if (ret)
9352                 goto disable_vreg;
9353
9354         /* enable the host irq as host controller would be active soon */
9355         ufshcd_enable_irq(hba);
9356         goto out;
9357
9358 disable_vreg:
9359         ufshcd_vreg_set_lpm(hba);
9360 out:
9361         if (ret)
9362                 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
9363         return ret;
9364 }
9365 #endif /* CONFIG_PM */
9366
9367 #ifdef CONFIG_PM_SLEEP
9368 /**
9369  * ufshcd_system_suspend - system suspend callback
9370  * @dev: Device associated with the UFS controller.
9371  *
9372  * Executed before putting the system into a sleep state in which the contents
9373  * of main memory are preserved.
9374  *
9375  * Returns 0 for success and non-zero for failure
9376  */
9377 int ufshcd_system_suspend(struct device *dev)
9378 {
9379         struct ufs_hba *hba = dev_get_drvdata(dev);
9380         int ret = 0;
9381         ktime_t start = ktime_get();
9382
9383         if (pm_runtime_suspended(hba->dev))
9384                 goto out;
9385
9386         ret = ufshcd_suspend(hba);
9387 out:
9388         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
9389                 ktime_to_us(ktime_sub(ktime_get(), start)),
9390                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9391         return ret;
9392 }
9393 EXPORT_SYMBOL(ufshcd_system_suspend);
9394
9395 /**
9396  * ufshcd_system_resume - system resume callback
9397  * @dev: Device associated with the UFS controller.
9398  *
9399  * Executed after waking the system up from a sleep state in which the contents
9400  * of main memory were preserved.
9401  *
9402  * Returns 0 for success and non-zero for failure
9403  */
9404 int ufshcd_system_resume(struct device *dev)
9405 {
9406         struct ufs_hba *hba = dev_get_drvdata(dev);
9407         ktime_t start = ktime_get();
9408         int ret = 0;
9409
9410         if (pm_runtime_suspended(hba->dev))
9411                 goto out;
9412
9413         ret = ufshcd_resume(hba);
9414
9415 out:
9416         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9417                 ktime_to_us(ktime_sub(ktime_get(), start)),
9418                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9419
9420         return ret;
9421 }
9422 EXPORT_SYMBOL(ufshcd_system_resume);
9423 #endif /* CONFIG_PM_SLEEP */
9424
9425 #ifdef CONFIG_PM
9426 /**
9427  * ufshcd_runtime_suspend - runtime suspend callback
9428  * @dev: Device associated with the UFS controller.
9429  *
9430  * Check the description of ufshcd_suspend() function for more details.
9431  *
9432  * Returns 0 for success and non-zero for failure
9433  */
9434 int ufshcd_runtime_suspend(struct device *dev)
9435 {
9436         struct ufs_hba *hba = dev_get_drvdata(dev);
9437         int ret;
9438         ktime_t start = ktime_get();
9439
9440         ret = ufshcd_suspend(hba);
9441
9442         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9443                 ktime_to_us(ktime_sub(ktime_get(), start)),
9444                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9445         return ret;
9446 }
9447 EXPORT_SYMBOL(ufshcd_runtime_suspend);
9448
9449 /**
9450  * ufshcd_runtime_resume - runtime resume routine
9451  * @dev: Device associated with the UFS controller.
9452  *
9453  * This function basically brings controller
9454  * to active state. Following operations are done in this function:
9455  *
9456  * 1. Turn on all the controller related clocks
9457  * 2. Turn ON VCC rail
9458  */
9459 int ufshcd_runtime_resume(struct device *dev)
9460 {
9461         struct ufs_hba *hba = dev_get_drvdata(dev);
9462         int ret;
9463         ktime_t start = ktime_get();
9464
9465         ret = ufshcd_resume(hba);
9466
9467         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9468                 ktime_to_us(ktime_sub(ktime_get(), start)),
9469                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9470         return ret;
9471 }
9472 EXPORT_SYMBOL(ufshcd_runtime_resume);
9473 #endif /* CONFIG_PM */
9474
9475 /**
9476  * ufshcd_shutdown - shutdown routine
9477  * @hba: per adapter instance
9478  *
9479  * This function would turn off both UFS device and UFS hba
9480  * regulators. It would also disable clocks.
9481  *
9482  * Returns 0 always to allow force shutdown even in case of errors.
9483  */
9484 int ufshcd_shutdown(struct ufs_hba *hba)
9485 {
9486         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9487                 goto out;
9488
9489         pm_runtime_get_sync(hba->dev);
9490
9491         ufshcd_suspend(hba);
9492 out:
9493         hba->is_powered = false;
9494         /* allow force shutdown even in case of errors */
9495         return 0;
9496 }
9497 EXPORT_SYMBOL(ufshcd_shutdown);
9498
9499 /**
9500  * ufshcd_remove - de-allocate SCSI host and host memory space
9501  *              data structure memory
9502  * @hba: per adapter instance
9503  */
9504 void ufshcd_remove(struct ufs_hba *hba)
9505 {
9506         if (hba->ufs_device_wlun)
9507                 ufshcd_rpm_get_sync(hba);
9508         ufs_hwmon_remove(hba);
9509         ufs_bsg_remove(hba);
9510         ufshpb_remove(hba);
9511         ufs_sysfs_remove_nodes(hba->dev);
9512         blk_cleanup_queue(hba->tmf_queue);
9513         blk_mq_free_tag_set(&hba->tmf_tag_set);
9514         scsi_remove_host(hba->host);
9515         /* disable interrupts */
9516         ufshcd_disable_intr(hba, hba->intr_mask);
9517         ufshcd_hba_stop(hba);
9518         ufshcd_hba_exit(hba);
9519 }
9520 EXPORT_SYMBOL_GPL(ufshcd_remove);
9521
9522 /**
9523  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9524  * @hba: pointer to Host Bus Adapter (HBA)
9525  */
9526 void ufshcd_dealloc_host(struct ufs_hba *hba)
9527 {
9528         scsi_host_put(hba->host);
9529 }
9530 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9531
9532 /**
9533  * ufshcd_set_dma_mask - Set dma mask based on the controller
9534  *                       addressing capability
9535  * @hba: per adapter instance
9536  *
9537  * Returns 0 for success, non-zero for failure
9538  */
9539 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9540 {
9541         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9542                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9543                         return 0;
9544         }
9545         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9546 }
9547
9548 /**
9549  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
9550  * @dev: pointer to device handle
9551  * @hba_handle: driver private handle
9552  * Returns 0 on success, non-zero value on failure
9553  */
9554 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
9555 {
9556         struct Scsi_Host *host;
9557         struct ufs_hba *hba;
9558         int err = 0;
9559
9560         if (!dev) {
9561                 dev_err(dev,
9562                 "Invalid memory reference for dev is NULL\n");
9563                 err = -ENODEV;
9564                 goto out_error;
9565         }
9566
9567         host = scsi_host_alloc(&ufshcd_driver_template,
9568                                 sizeof(struct ufs_hba));
9569         if (!host) {
9570                 dev_err(dev, "scsi_host_alloc failed\n");
9571                 err = -ENOMEM;
9572                 goto out_error;
9573         }
9574         host->nr_maps = HCTX_TYPE_POLL + 1;
9575         hba = shost_priv(host);
9576         hba->host = host;
9577         hba->dev = dev;
9578         hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
9579         hba->nop_out_timeout = NOP_OUT_TIMEOUT;
9580         INIT_LIST_HEAD(&hba->clk_list_head);
9581         spin_lock_init(&hba->outstanding_lock);
9582
9583         *hba_handle = hba;
9584
9585 out_error:
9586         return err;
9587 }
9588 EXPORT_SYMBOL(ufshcd_alloc_host);
9589
9590 /* This function exists because blk_mq_alloc_tag_set() requires this. */
9591 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9592                                      const struct blk_mq_queue_data *qd)
9593 {
9594         WARN_ON_ONCE(true);
9595         return BLK_STS_NOTSUPP;
9596 }
9597
9598 static const struct blk_mq_ops ufshcd_tmf_ops = {
9599         .queue_rq = ufshcd_queue_tmf,
9600 };
9601
9602 /**
9603  * ufshcd_init - Driver initialization routine
9604  * @hba: per-adapter instance
9605  * @mmio_base: base register address
9606  * @irq: Interrupt line of device
9607  * Returns 0 on success, non-zero value on failure
9608  */
9609 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9610 {
9611         int err;
9612         struct Scsi_Host *host = hba->host;
9613         struct device *dev = hba->dev;
9614         char eh_wq_name[sizeof("ufs_eh_wq_00")];
9615
9616         /*
9617          * dev_set_drvdata() must be called before any callbacks are registered
9618          * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
9619          * sysfs).
9620          */
9621         dev_set_drvdata(dev, hba);
9622
9623         if (!mmio_base) {
9624                 dev_err(hba->dev,
9625                 "Invalid memory reference for mmio_base is NULL\n");
9626                 err = -ENODEV;
9627                 goto out_error;
9628         }
9629
9630         hba->mmio_base = mmio_base;
9631         hba->irq = irq;
9632         hba->vps = &ufs_hba_vps;
9633
9634         err = ufshcd_hba_init(hba);
9635         if (err)
9636                 goto out_error;
9637
9638         /* Read capabilities registers */
9639         err = ufshcd_hba_capabilities(hba);
9640         if (err)
9641                 goto out_disable;
9642
9643         /* Get UFS version supported by the controller */
9644         hba->ufs_version = ufshcd_get_ufs_version(hba);
9645
9646         /* Get Interrupt bit mask per version */
9647         hba->intr_mask = ufshcd_get_intr_mask(hba);
9648
9649         err = ufshcd_set_dma_mask(hba);
9650         if (err) {
9651                 dev_err(hba->dev, "set dma mask failed\n");
9652                 goto out_disable;
9653         }
9654
9655         /* Allocate memory for host memory space */
9656         err = ufshcd_memory_alloc(hba);
9657         if (err) {
9658                 dev_err(hba->dev, "Memory allocation failed\n");
9659                 goto out_disable;
9660         }
9661
9662         /* Configure LRB */
9663         ufshcd_host_memory_configure(hba);
9664
9665         host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
9666         host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
9667         host->max_id = UFSHCD_MAX_ID;
9668         host->max_lun = UFS_MAX_LUNS;
9669         host->max_channel = UFSHCD_MAX_CHANNEL;
9670         host->unique_id = host->host_no;
9671         host->max_cmd_len = UFS_CDB_SIZE;
9672
9673         hba->max_pwr_info.is_valid = false;
9674
9675         /* Initialize work queues */
9676         snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9677                  hba->host->host_no);
9678         hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9679         if (!hba->eh_wq) {
9680                 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9681                         __func__);
9682                 err = -ENOMEM;
9683                 goto out_disable;
9684         }
9685         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
9686         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
9687
9688         sema_init(&hba->host_sem, 1);
9689
9690         /* Initialize UIC command mutex */
9691         mutex_init(&hba->uic_cmd_mutex);
9692
9693         /* Initialize mutex for device management commands */
9694         mutex_init(&hba->dev_cmd.lock);
9695
9696         /* Initialize mutex for exception event control */
9697         mutex_init(&hba->ee_ctrl_mutex);
9698
9699         init_rwsem(&hba->clk_scaling_lock);
9700
9701         ufshcd_init_clk_gating(hba);
9702
9703         ufshcd_init_clk_scaling(hba);
9704
9705         /*
9706          * In order to avoid any spurious interrupt immediately after
9707          * registering UFS controller interrupt handler, clear any pending UFS
9708          * interrupt status and disable all the UFS interrupts.
9709          */
9710         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9711                       REG_INTERRUPT_STATUS);
9712         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9713         /*
9714          * Make sure that UFS interrupts are disabled and any pending interrupt
9715          * status is cleared before registering UFS interrupt handler.
9716          */
9717         mb();
9718
9719         /* IRQ registration */
9720         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
9721         if (err) {
9722                 dev_err(hba->dev, "request irq failed\n");
9723                 goto out_disable;
9724         } else {
9725                 hba->is_irq_enabled = true;
9726         }
9727
9728         err = scsi_add_host(host, hba->dev);
9729         if (err) {
9730                 dev_err(hba->dev, "scsi_add_host failed\n");
9731                 goto out_disable;
9732         }
9733
9734         hba->tmf_tag_set = (struct blk_mq_tag_set) {
9735                 .nr_hw_queues   = 1,
9736                 .queue_depth    = hba->nutmrs,
9737                 .ops            = &ufshcd_tmf_ops,
9738                 .flags          = BLK_MQ_F_NO_SCHED,
9739         };
9740         err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9741         if (err < 0)
9742                 goto out_remove_scsi_host;
9743         hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9744         if (IS_ERR(hba->tmf_queue)) {
9745                 err = PTR_ERR(hba->tmf_queue);
9746                 goto free_tmf_tag_set;
9747         }
9748         hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
9749                                     sizeof(*hba->tmf_rqs), GFP_KERNEL);
9750         if (!hba->tmf_rqs) {
9751                 err = -ENOMEM;
9752                 goto free_tmf_queue;
9753         }
9754
9755         /* Reset the attached device */
9756         ufshcd_device_reset(hba);
9757
9758         ufshcd_init_crypto(hba);
9759
9760         /* Host controller enable */
9761         err = ufshcd_hba_enable(hba);
9762         if (err) {
9763                 dev_err(hba->dev, "Host controller enable failed\n");
9764                 ufshcd_print_evt_hist(hba);
9765                 ufshcd_print_host_state(hba);
9766                 goto free_tmf_queue;
9767         }
9768
9769         /*
9770          * Set the default power management level for runtime and system PM.
9771          * Default power saving mode is to keep UFS link in Hibern8 state
9772          * and UFS device in sleep state.
9773          */
9774         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9775                                                 UFS_SLEEP_PWR_MODE,
9776                                                 UIC_LINK_HIBERN8_STATE);
9777         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9778                                                 UFS_SLEEP_PWR_MODE,
9779                                                 UIC_LINK_HIBERN8_STATE);
9780
9781         INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9782                           ufshcd_rpm_dev_flush_recheck_work);
9783
9784         /* Set the default auto-hiberate idle timer value to 150 ms */
9785         if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
9786                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9787                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9788         }
9789
9790         /* Hold auto suspend until async scan completes */
9791         pm_runtime_get_sync(dev);
9792         atomic_set(&hba->scsi_block_reqs_cnt, 0);
9793         /*
9794          * We are assuming that device wasn't put in sleep/power-down
9795          * state exclusively during the boot stage before kernel.
9796          * This assumption helps avoid doing link startup twice during
9797          * ufshcd_probe_hba().
9798          */
9799         ufshcd_set_ufs_dev_active(hba);
9800
9801         async_schedule(ufshcd_async_scan, hba);
9802         ufs_sysfs_add_nodes(hba->dev);
9803
9804         device_enable_async_suspend(dev);
9805         return 0;
9806
9807 free_tmf_queue:
9808         blk_cleanup_queue(hba->tmf_queue);
9809 free_tmf_tag_set:
9810         blk_mq_free_tag_set(&hba->tmf_tag_set);
9811 out_remove_scsi_host:
9812         scsi_remove_host(hba->host);
9813 out_disable:
9814         hba->is_irq_enabled = false;
9815         ufshcd_hba_exit(hba);
9816 out_error:
9817         return err;
9818 }
9819 EXPORT_SYMBOL_GPL(ufshcd_init);
9820
9821 void ufshcd_resume_complete(struct device *dev)
9822 {
9823         struct ufs_hba *hba = dev_get_drvdata(dev);
9824
9825         if (hba->complete_put) {
9826                 ufshcd_rpm_put(hba);
9827                 hba->complete_put = false;
9828         }
9829 }
9830 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
9831
9832 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
9833 {
9834         struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
9835         enum ufs_dev_pwr_mode dev_pwr_mode;
9836         enum uic_link_state link_state;
9837         unsigned long flags;
9838         bool res;
9839
9840         spin_lock_irqsave(&dev->power.lock, flags);
9841         dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
9842         link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
9843         res = pm_runtime_suspended(dev) &&
9844               hba->curr_dev_pwr_mode == dev_pwr_mode &&
9845               hba->uic_link_state == link_state &&
9846               !hba->dev_info.b_rpm_dev_flush_capable;
9847         spin_unlock_irqrestore(&dev->power.lock, flags);
9848
9849         return res;
9850 }
9851
9852 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
9853 {
9854         struct ufs_hba *hba = dev_get_drvdata(dev);
9855         int ret;
9856
9857         /*
9858          * SCSI assumes that runtime-pm and system-pm for scsi drivers
9859          * are same. And it doesn't wake up the device for system-suspend
9860          * if it's runtime suspended. But ufs doesn't follow that.
9861          * Refer ufshcd_resume_complete()
9862          */
9863         if (hba->ufs_device_wlun) {
9864                 /* Prevent runtime suspend */
9865                 ufshcd_rpm_get_noresume(hba);
9866                 /*
9867                  * Check if already runtime suspended in same state as system
9868                  * suspend would be.
9869                  */
9870                 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
9871                         /* RPM state is not ok for SPM, so runtime resume */
9872                         ret = ufshcd_rpm_resume(hba);
9873                         if (ret < 0 && ret != -EACCES) {
9874                                 ufshcd_rpm_put(hba);
9875                                 return ret;
9876                         }
9877                 }
9878                 hba->complete_put = true;
9879         }
9880         return 0;
9881 }
9882 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
9883
9884 int ufshcd_suspend_prepare(struct device *dev)
9885 {
9886         return __ufshcd_suspend_prepare(dev, true);
9887 }
9888 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
9889
9890 #ifdef CONFIG_PM_SLEEP
9891 static int ufshcd_wl_poweroff(struct device *dev)
9892 {
9893         struct scsi_device *sdev = to_scsi_device(dev);
9894         struct ufs_hba *hba = shost_priv(sdev->host);
9895
9896         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9897         return 0;
9898 }
9899 #endif
9900
9901 static int ufshcd_wl_probe(struct device *dev)
9902 {
9903         struct scsi_device *sdev = to_scsi_device(dev);
9904
9905         if (!is_device_wlun(sdev))
9906                 return -ENODEV;
9907
9908         blk_pm_runtime_init(sdev->request_queue, dev);
9909         pm_runtime_set_autosuspend_delay(dev, 0);
9910         pm_runtime_allow(dev);
9911
9912         return  0;
9913 }
9914
9915 static int ufshcd_wl_remove(struct device *dev)
9916 {
9917         pm_runtime_forbid(dev);
9918         return 0;
9919 }
9920
9921 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
9922 #ifdef CONFIG_PM_SLEEP
9923         .suspend = ufshcd_wl_suspend,
9924         .resume = ufshcd_wl_resume,
9925         .freeze = ufshcd_wl_suspend,
9926         .thaw = ufshcd_wl_resume,
9927         .poweroff = ufshcd_wl_poweroff,
9928         .restore = ufshcd_wl_resume,
9929 #endif
9930         SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
9931 };
9932
9933 /*
9934  * ufs_dev_wlun_template - describes ufs device wlun
9935  * ufs-device wlun - used to send pm commands
9936  * All luns are consumers of ufs-device wlun.
9937  *
9938  * Currently, no sd driver is present for wluns.
9939  * Hence the no specific pm operations are performed.
9940  * With ufs design, SSU should be sent to ufs-device wlun.
9941  * Hence register a scsi driver for ufs wluns only.
9942  */
9943 static struct scsi_driver ufs_dev_wlun_template = {
9944         .gendrv = {
9945                 .name = "ufs_device_wlun",
9946                 .owner = THIS_MODULE,
9947                 .probe = ufshcd_wl_probe,
9948                 .remove = ufshcd_wl_remove,
9949                 .pm = &ufshcd_wl_pm_ops,
9950                 .shutdown = ufshcd_wl_shutdown,
9951         },
9952 };
9953
9954 static int __init ufshcd_core_init(void)
9955 {
9956         int ret;
9957
9958         /* Verify that there are no gaps in struct utp_transfer_cmd_desc. */
9959         static_assert(sizeof(struct utp_transfer_cmd_desc) ==
9960                       2 * ALIGNED_UPIU_SIZE +
9961                               SG_ALL * sizeof(struct ufshcd_sg_entry));
9962
9963         ufs_debugfs_init();
9964
9965         ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
9966         if (ret)
9967                 ufs_debugfs_exit();
9968         return ret;
9969 }
9970
9971 static void __exit ufshcd_core_exit(void)
9972 {
9973         ufs_debugfs_exit();
9974         scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
9975 }
9976
9977 module_init(ufshcd_core_init);
9978 module_exit(ufshcd_core_exit);
9979
9980 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9981 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
9982 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
9983 MODULE_LICENSE("GPL");