drivers/ufs/core/ufshcd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Universal Flash Storage Host controller driver Core
   4  * Copyright (C) 2011-2013 Samsung India Software Operations
   5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   6  *
   7  * Authors:
   8  *      Santosh Yaraganavi <santosh.sy@samsung.com>
   9  *      Vinayak Holikatti <h.vinayak@samsung.com>
  10  */
  11
  12 #include <linux/async.h>
  13 #include <linux/devfreq.h>
  14 #include <linux/nls.h>
  15 #include <linux/of.h>
  16 #include <linux/bitfield.h>
  17 #include <linux/blk-pm.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/clk.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/module.h>
  23 #include <linux/pm_opp.h>
  24 #include <linux/regulator/consumer.h>
  25 #include <linux/sched/clock.h>
  26 #include <linux/iopoll.h>
  27 #include <scsi/scsi_cmnd.h>
  28 #include <scsi/scsi_dbg.h>
  29 #include <scsi/scsi_driver.h>
  30 #include <scsi/scsi_eh.h>
  31 #include "ufshcd-priv.h"
  32 #include <ufs/ufs_quirks.h>
  33 #include <ufs/unipro.h>
  34 #include "ufs-sysfs.h"
  35 #include "ufs-debugfs.h"
  36 #include "ufs-fault-injection.h"
  37 #include "ufs_bsg.h"
  38 #include "ufshcd-crypto.h"
  39 #include <asm/unaligned.h>
  40
  41 #define CREATE_TRACE_POINTS
  42 #include <trace/events/ufs.h>
  43
  44 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  45                                  UTP_TASK_REQ_COMPL |\
  46                                  UFSHCD_ERROR_MASK)
  47
  48 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\
  49                                  UFSHCD_ERROR_MASK |\
  50                                  MCQ_CQ_EVENT_STATUS)
  51
  52
  53 /* UIC command timeout, unit: ms */
  54 #define UIC_CMD_TIMEOUT 500
  55
  56 /* NOP OUT retries waiting for NOP IN response */
  57 #define NOP_OUT_RETRIES    10
  58 /* Timeout after 50 msecs if NOP OUT hangs without response */
  59 #define NOP_OUT_TIMEOUT    50 /* msecs */
  60
  61 /* Query request retries */
  62 #define QUERY_REQ_RETRIES 3
  63 /* Query request timeout */
  64 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  65
  66 /* Advanced RPMB request timeout */
  67 #define ADVANCED_RPMB_REQ_TIMEOUT  3000 /* 3 seconds */
  68
  69 /* Task management command timeout */
  70 #define TM_CMD_TIMEOUT  100 /* msecs */
  71
  72 /* maximum number of retries for a general UIC command  */
  73 #define UFS_UIC_COMMAND_RETRIES 3
  74
  75 /* maximum number of link-startup retries */
  76 #define DME_LINKSTARTUP_RETRIES 3
  77
  78 /* maximum number of reset retries before giving up */
  79 #define MAX_HOST_RESET_RETRIES 5
  80
  81 /* Maximum number of error handler retries before giving up */
  82 #define MAX_ERR_HANDLER_RETRIES 5
  83
  84 /* Expose the flag value from utp_upiu_query.value */
  85 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  86
  87 /* Interrupt aggregation default timeout, unit: 40us */
  88 #define INT_AGGR_DEF_TO 0x02
  89
  90 /* default delay of autosuspend: 2000 ms */
  91 #define RPM_AUTOSUSPEND_DELAY_MS 2000
  92
  93 /* Default delay of RPM device flush delayed work */
  94 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
  95
  96 /* Default value of wait time before gating device ref clock */
  97 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
  98
  99 /* Polling time to wait for fDeviceInit */
 100 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
 101
 102 /* Default RTC update every 10 seconds */
 103 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC)
 104
 105 /* UFSHC 4.0 compliant HC support this mode. */
 106 static bool use_mcq_mode = true;
 107
 108 static bool is_mcq_supported(struct ufs_hba *hba)
 109 {
 110         return hba->mcq_sup && use_mcq_mode;
 111 }
 112
 113 module_param(use_mcq_mode, bool, 0644);
 114 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
 115
 116 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
 117         ({                                                              \
 118                 int _ret;                                               \
 119                 if (_on)                                                \
 120                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
 121                 else                                                    \
 122                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
 123                 _ret;                                                   \
 124         })
 125
 126 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
 127         size_t __len = (len);                                            \
 128         print_hex_dump(KERN_ERR, prefix_str,                             \
 129                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
 130                        16, 4, buf, __len, false);                        \
 131 } while (0)
 132
 133 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 134                      const char *prefix)
 135 {
 136         u32 *regs;
 137         size_t pos;
 138
 139         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 140                 return -EINVAL;
 141
 142         regs = kzalloc(len, GFP_ATOMIC);
 143         if (!regs)
 144                 return -ENOMEM;
 145
 146         for (pos = 0; pos < len; pos += 4) {
 147                 if (offset == 0 &&
 148                     pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
 149                     pos <= REG_UIC_ERROR_CODE_DME)
 150                         continue;
 151                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 152         }
 153
 154         ufshcd_hex_dump(prefix, regs, len);
 155         kfree(regs);
 156
 157         return 0;
 158 }
 159 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 160
 161 enum {
 162         UFSHCD_MAX_CHANNEL      = 0,
 163         UFSHCD_MAX_ID           = 1,
 164         UFSHCD_CMD_PER_LUN      = 32 - UFSHCD_NUM_RESERVED,
 165         UFSHCD_CAN_QUEUE        = 32 - UFSHCD_NUM_RESERVED,
 166 };
 167
 168 static const char *const ufshcd_state_name[] = {
 169         [UFSHCD_STATE_RESET]                    = "reset",
 170         [UFSHCD_STATE_OPERATIONAL]              = "operational",
 171         [UFSHCD_STATE_ERROR]                    = "error",
 172         [UFSHCD_STATE_EH_SCHEDULED_FATAL]       = "eh_fatal",
 173         [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL]   = "eh_non_fatal",
 174 };
 175
 176 /* UFSHCD error handling flags */
 177 enum {
 178         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 179 };
 180
 181 /* UFSHCD UIC layer error flags */
 182 enum {
 183         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 184         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 185         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 186         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 187         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 188         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 189         UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
 190 };
 191
 192 #define ufshcd_set_eh_in_progress(h) \
 193         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 194 #define ufshcd_eh_in_progress(h) \
 195         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 196 #define ufshcd_clear_eh_in_progress(h) \
 197         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 198
 199 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 200         [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 201         [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 202         [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 203         [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 204         [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 205         [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 206         /*
 207          * For DeepSleep, the link is first put in hibern8 and then off.
 208          * Leaving the link in hibern8 is not supported.
 209          */
 210         [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
 211 };
 212
 213 static inline enum ufs_dev_pwr_mode
 214 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 215 {
 216         return ufs_pm_lvl_states[lvl].dev_state;
 217 }
 218
 219 static inline enum uic_link_state
 220 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 221 {
 222         return ufs_pm_lvl_states[lvl].link_state;
 223 }
 224
 225 static inline enum ufs_pm_level
 226 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 227                                         enum uic_link_state link_state)
 228 {
 229         enum ufs_pm_level lvl;
 230
 231         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 232                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 233                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 234                         return lvl;
 235         }
 236
 237         /* if no match found, return the level 0 */
 238         return UFS_PM_LVL_0;
 239 }
 240
 241 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba)
 242 {
 243         return (hba->clk_gating.active_reqs || hba->outstanding_reqs || hba->outstanding_tasks ||
 244                 hba->active_uic_cmd || hba->uic_async_done);
 245 }
 246
 247 static const struct ufs_dev_quirk ufs_fixups[] = {
 248         /* UFS cards deviations table */
 249         { .wmanufacturerid = UFS_VENDOR_MICRON,
 250           .model = UFS_ANY_MODEL,
 251           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
 252         { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
 253           .model = UFS_ANY_MODEL,
 254           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 255                    UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
 256                    UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
 257         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 258           .model = UFS_ANY_MODEL,
 259           .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
 260         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 261           .model = "hB8aL1" /*H28U62301AMR*/,
 262           .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
 263         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 264           .model = UFS_ANY_MODEL,
 265           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
 266         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 267           .model = "THGLF2G9C8KBADG",
 268           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 269         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 270           .model = "THGLF2G9D8KBADG",
 271           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 272         {}
 273 };
 274
 275 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 276 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 277 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 278 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 279 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 280 static void ufshcd_hba_exit(struct ufs_hba *hba);
 281 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
 282 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 283 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 284 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 285 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 286 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 287 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
 288                              bool scale_up);
 289 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 290 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 291                              struct ufs_pa_layer_attr *pwr_mode);
 292 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 293 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 294 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 295                                          struct ufs_vreg *vreg);
 296 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
 297                                                  bool enable);
 298 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
 299 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
 300
 301 void ufshcd_enable_irq(struct ufs_hba *hba)
 302 {
 303         if (!hba->is_irq_enabled) {
 304                 enable_irq(hba->irq);
 305                 hba->is_irq_enabled = true;
 306         }
 307 }
 308 EXPORT_SYMBOL_GPL(ufshcd_enable_irq);
 309
 310 void ufshcd_disable_irq(struct ufs_hba *hba)
 311 {
 312         if (hba->is_irq_enabled) {
 313                 disable_irq(hba->irq);
 314                 hba->is_irq_enabled = false;
 315         }
 316 }
 317 EXPORT_SYMBOL_GPL(ufshcd_disable_irq);
 318
 319 static void ufshcd_configure_wb(struct ufs_hba *hba)
 320 {
 321         if (!ufshcd_is_wb_allowed(hba))
 322                 return;
 323
 324         ufshcd_wb_toggle(hba, true);
 325
 326         ufshcd_wb_toggle_buf_flush_during_h8(hba, true);
 327
 328         if (ufshcd_is_wb_buf_flush_allowed(hba))
 329                 ufshcd_wb_toggle_buf_flush(hba, true);
 330 }
 331
 332 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 333 {
 334         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 335                 scsi_unblock_requests(hba->host);
 336 }
 337
 338 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 339 {
 340         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 341                 scsi_block_requests(hba->host);
 342 }
 343
 344 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 345                                       enum ufs_trace_str_t str_t)
 346 {
 347         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 348         struct utp_upiu_header *header;
 349
 350         if (!trace_ufshcd_upiu_enabled())
 351                 return;
 352
 353         if (str_t == UFS_CMD_SEND)
 354                 header = &rq->header;
 355         else
 356                 header = &hba->lrb[tag].ucd_rsp_ptr->header;
 357
 358         trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
 359                           UFS_TSF_CDB);
 360 }
 361
 362 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
 363                                         enum ufs_trace_str_t str_t,
 364                                         struct utp_upiu_req *rq_rsp)
 365 {
 366         if (!trace_ufshcd_upiu_enabled())
 367                 return;
 368
 369         trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
 370                           &rq_rsp->qr, UFS_TSF_OSF);
 371 }
 372
 373 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 374                                      enum ufs_trace_str_t str_t)
 375 {
 376         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
 377
 378         if (!trace_ufshcd_upiu_enabled())
 379                 return;
 380
 381         if (str_t == UFS_TM_SEND)
 382                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 383                                   &descp->upiu_req.req_header,
 384                                   &descp->upiu_req.input_param1,
 385                                   UFS_TSF_TM_INPUT);
 386         else
 387                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 388                                   &descp->upiu_rsp.rsp_header,
 389                                   &descp->upiu_rsp.output_param1,
 390                                   UFS_TSF_TM_OUTPUT);
 391 }
 392
 393 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
 394                                          const struct uic_command *ucmd,
 395                                          enum ufs_trace_str_t str_t)
 396 {
 397         u32 cmd;
 398
 399         if (!trace_ufshcd_uic_command_enabled())
 400                 return;
 401
 402         if (str_t == UFS_CMD_SEND)
 403                 cmd = ucmd->command;
 404         else
 405                 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
 406
 407         trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
 408                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
 409                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
 410                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
 411 }
 412
 413 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 414                                      enum ufs_trace_str_t str_t)
 415 {
 416         u64 lba = 0;
 417         u8 opcode = 0, group_id = 0;
 418         u32 doorbell = 0;
 419         u32 intr;
 420         int hwq_id = -1;
 421         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 422         struct scsi_cmnd *cmd = lrbp->cmd;
 423         struct request *rq = scsi_cmd_to_rq(cmd);
 424         int transfer_len = -1;
 425
 426         if (!cmd)
 427                 return;
 428
 429         /* trace UPIU also */
 430         ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 431         if (!trace_ufshcd_command_enabled())
 432                 return;
 433
 434         opcode = cmd->cmnd[0];
 435
 436         if (opcode == READ_10 || opcode == WRITE_10) {
 437                 /*
 438                  * Currently we only fully trace read(10) and write(10) commands
 439                  */
 440                 transfer_len =
 441                        be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 442                 lba = scsi_get_lba(cmd);
 443                 if (opcode == WRITE_10)
 444                         group_id = lrbp->cmd->cmnd[6];
 445         } else if (opcode == UNMAP) {
 446                 /*
 447                  * The number of Bytes to be unmapped beginning with the lba.
 448                  */
 449                 transfer_len = blk_rq_bytes(rq);
 450                 lba = scsi_get_lba(cmd);
 451         }
 452
 453         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 454
 455         if (is_mcq_enabled(hba)) {
 456                 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
 457
 458                 hwq_id = hwq->id;
 459         } else {
 460                 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 461         }
 462         trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id,
 463                              transfer_len, intr, lba, opcode, group_id);
 464 }
 465
 466 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 467 {
 468         struct ufs_clk_info *clki;
 469         struct list_head *head = &hba->clk_list_head;
 470
 471         if (list_empty(head))
 472                 return;
 473
 474         list_for_each_entry(clki, head, list) {
 475                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 476                                 clki->max_freq)
 477                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 478                                         clki->name, clki->curr_freq);
 479         }
 480 }
 481
 482 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
 483                              const char *err_name)
 484 {
 485         int i;
 486         bool found = false;
 487         const struct ufs_event_hist *e;
 488
 489         if (id >= UFS_EVT_CNT)
 490                 return;
 491
 492         e = &hba->ufs_stats.event[id];
 493
 494         for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
 495                 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
 496
 497                 if (e->tstamp[p] == 0)
 498                         continue;
 499                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 500                         e->val[p], div_u64(e->tstamp[p], 1000));
 501                 found = true;
 502         }
 503
 504         if (!found)
 505                 dev_err(hba->dev, "No record of %s\n", err_name);
 506         else
 507                 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
 508 }
 509
 510 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 511 {
 512         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 513
 514         ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
 515         ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
 516         ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
 517         ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
 518         ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
 519         ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
 520                          "auto_hibern8_err");
 521         ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
 522         ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
 523                          "link_startup_fail");
 524         ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
 525         ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
 526                          "suspend_fail");
 527         ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail");
 528         ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR,
 529                          "wlun suspend_fail");
 530         ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
 531         ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
 532         ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
 533
 534         ufshcd_vops_dbg_register_dump(hba);
 535 }
 536
 537 static
 538 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
 539 {
 540         const struct ufshcd_lrb *lrbp;
 541         int prdt_length;
 542
 543         lrbp = &hba->lrb[tag];
 544
 545         dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 546                         tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
 547         dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 548                         tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
 549         dev_err(hba->dev,
 550                 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 551                 tag, (u64)lrbp->utrd_dma_addr);
 552
 553         ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 554                         sizeof(struct utp_transfer_req_desc));
 555         dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 556                 (u64)lrbp->ucd_req_dma_addr);
 557         ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 558                         sizeof(struct utp_upiu_req));
 559         dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 560                 (u64)lrbp->ucd_rsp_dma_addr);
 561         ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 562                         sizeof(struct utp_upiu_rsp));
 563
 564         prdt_length = le16_to_cpu(
 565                 lrbp->utr_descriptor_ptr->prd_table_length);
 566         if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
 567                 prdt_length /= ufshcd_sg_entry_size(hba);
 568
 569         dev_err(hba->dev,
 570                 "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 571                 tag, prdt_length,
 572                 (u64)lrbp->ucd_prdt_dma_addr);
 573
 574         if (pr_prdt)
 575                 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 576                         ufshcd_sg_entry_size(hba) * prdt_length);
 577 }
 578
 579 static bool ufshcd_print_tr_iter(struct request *req, void *priv)
 580 {
 581         struct scsi_device *sdev = req->q->queuedata;
 582         struct Scsi_Host *shost = sdev->host;
 583         struct ufs_hba *hba = shost_priv(shost);
 584
 585         ufshcd_print_tr(hba, req->tag, *(bool *)priv);
 586
 587         return true;
 588 }
 589
 590 /**
 591  * ufshcd_print_trs_all - print trs for all started requests.
 592  * @hba: per-adapter instance.
 593  * @pr_prdt: need to print prdt or not.
 594  */
 595 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt)
 596 {
 597         blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
 598 }
 599
 600 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 601 {
 602         int tag;
 603
 604         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 605                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 606
 607                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 608                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 609         }
 610 }
 611
 612 static void ufshcd_print_host_state(struct ufs_hba *hba)
 613 {
 614         const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
 615
 616         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 617         dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 618                 hba->outstanding_reqs, hba->outstanding_tasks);
 619         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 620                 hba->saved_err, hba->saved_uic_err);
 621         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 622                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 623         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 624                 hba->pm_op_in_progress, hba->is_sys_suspended);
 625         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 626                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 627         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 628         dev_err(hba->dev,
 629                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
 630                 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
 631                 hba->ufs_stats.hibern8_exit_cnt);
 632         dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
 633                 div_u64(hba->ufs_stats.last_intr_ts, 1000),
 634                 hba->ufs_stats.last_intr_status);
 635         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 636                 hba->eh_flags, hba->req_abort_count);
 637         dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
 638                 hba->ufs_version, hba->capabilities, hba->caps);
 639         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 640                 hba->dev_quirks);
 641         if (sdev_ufs)
 642                 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
 643                         sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
 644
 645         ufshcd_print_clk_freqs(hba);
 646 }
 647
 648 /**
 649  * ufshcd_print_pwr_info - print power params as saved in hba
 650  * power info
 651  * @hba: per-adapter instance
 652  */
 653 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 654 {
 655         static const char * const names[] = {
 656                 "INVALID MODE",
 657                 "FAST MODE",
 658                 "SLOW_MODE",
 659                 "INVALID MODE",
 660                 "FASTAUTO_MODE",
 661                 "SLOWAUTO_MODE",
 662                 "INVALID MODE",
 663         };
 664
 665         /*
 666          * Using dev_dbg to avoid messages during runtime PM to avoid
 667          * never-ending cycles of messages written back to storage by user space
 668          * causing runtime resume, causing more messages and so on.
 669          */
 670         dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 671                  __func__,
 672                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 673                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 674                  names[hba->pwr_info.pwr_rx],
 675                  names[hba->pwr_info.pwr_tx],
 676                  hba->pwr_info.hs_rate);
 677 }
 678
 679 static void ufshcd_device_reset(struct ufs_hba *hba)
 680 {
 681         int err;
 682
 683         err = ufshcd_vops_device_reset(hba);
 684
 685         if (!err) {
 686                 ufshcd_set_ufs_dev_active(hba);
 687                 if (ufshcd_is_wb_allowed(hba)) {
 688                         hba->dev_info.wb_enabled = false;
 689                         hba->dev_info.wb_buf_flush_enabled = false;
 690                 }
 691                 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
 692                         hba->dev_info.rtc_time_baseline = 0;
 693         }
 694         if (err != -EOPNOTSUPP)
 695                 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
 696 }
 697
 698 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 699 {
 700         if (!us)
 701                 return;
 702
 703         if (us < 10)
 704                 udelay(us);
 705         else
 706                 usleep_range(us, us + tolerance);
 707 }
 708 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 709
 710 /**
 711  * ufshcd_wait_for_register - wait for register value to change
 712  * @hba: per-adapter interface
 713  * @reg: mmio register offset
 714  * @mask: mask to apply to the read register value
 715  * @val: value to wait for
 716  * @interval_us: polling interval in microseconds
 717  * @timeout_ms: timeout in milliseconds
 718  *
 719  * Return: -ETIMEDOUT on error, zero on success.
 720  */
 721 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 722                                 u32 val, unsigned long interval_us,
 723                                 unsigned long timeout_ms)
 724 {
 725         int err = 0;
 726         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 727
 728         /* ignore bits that we don't intend to wait on */
 729         val = val & mask;
 730
 731         while ((ufshcd_readl(hba, reg) & mask) != val) {
 732                 usleep_range(interval_us, interval_us + 50);
 733                 if (time_after(jiffies, timeout)) {
 734                         if ((ufshcd_readl(hba, reg) & mask) != val)
 735                                 err = -ETIMEDOUT;
 736                         break;
 737                 }
 738         }
 739
 740         return err;
 741 }
 742
 743 /**
 744  * ufshcd_get_intr_mask - Get the interrupt bit mask
 745  * @hba: Pointer to adapter instance
 746  *
 747  * Return: interrupt bit mask per version
 748  */
 749 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 750 {
 751         if (hba->ufs_version == ufshci_version(1, 0))
 752                 return INTERRUPT_MASK_ALL_VER_10;
 753         if (hba->ufs_version <= ufshci_version(2, 0))
 754                 return INTERRUPT_MASK_ALL_VER_11;
 755
 756         return INTERRUPT_MASK_ALL_VER_21;
 757 }
 758
 759 /**
 760  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 761  * @hba: Pointer to adapter instance
 762  *
 763  * Return: UFSHCI version supported by the controller
 764  */
 765 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 766 {
 767         u32 ufshci_ver;
 768
 769         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 770                 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
 771         else
 772                 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
 773
 774         /*
 775          * UFSHCI v1.x uses a different version scheme, in order
 776          * to allow the use of comparisons with the ufshci_version
 777          * function, we convert it to the same scheme as ufs 2.0+.
 778          */
 779         if (ufshci_ver & 0x00010000)
 780                 return ufshci_version(1, ufshci_ver & 0x00000100);
 781
 782         return ufshci_ver;
 783 }
 784
 785 /**
 786  * ufshcd_is_device_present - Check if any device connected to
 787  *                            the host controller
 788  * @hba: pointer to adapter instance
 789  *
 790  * Return: true if device present, false if no device detected
 791  */
 792 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 793 {
 794         return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
 795 }
 796
 797 /**
 798  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 799  * @lrbp: pointer to local command reference block
 800  * @cqe: pointer to the completion queue entry
 801  *
 802  * This function is used to get the OCS field from UTRD
 803  *
 804  * Return: the OCS field in the UTRD.
 805  */
 806 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp,
 807                                       struct cq_entry *cqe)
 808 {
 809         if (cqe)
 810                 return le32_to_cpu(cqe->status) & MASK_OCS;
 811
 812         return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
 813 }
 814
 815 /**
 816  * ufshcd_utrl_clear() - Clear requests from the controller request list.
 817  * @hba: per adapter instance
 818  * @mask: mask with one bit set for each request to be cleared
 819  */
 820 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
 821 {
 822         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 823                 mask = ~mask;
 824         /*
 825          * From the UFSHCI specification: "UTP Transfer Request List CLear
 826          * Register (UTRLCLR): This field is bit significant. Each bit
 827          * corresponds to a slot in the UTP Transfer Request List, where bit 0
 828          * corresponds to request slot 0. A bit in this field is set to ‘0’
 829          * by host software to indicate to the host controller that a transfer
 830          * request slot is cleared. The host controller
 831          * shall free up any resources associated to the request slot
 832          * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
 833          * host software indicates no change to request slots by setting the
 834          * associated bits in this field to ‘1’. Bits in this field shall only
 835          * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
 836          */
 837         ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 838 }
 839
 840 /**
 841  * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
 842  * @hba: per adapter instance
 843  * @pos: position of the bit to be cleared
 844  */
 845 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 846 {
 847         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 848                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 849         else
 850                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 851 }
 852
 853 /**
 854  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 855  * @reg: Register value of host controller status
 856  *
 857  * Return: 0 on success; a positive value if failed.
 858  */
 859 static inline int ufshcd_get_lists_status(u32 reg)
 860 {
 861         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 862 }
 863
 864 /**
 865  * ufshcd_get_uic_cmd_result - Get the UIC command result
 866  * @hba: Pointer to adapter instance
 867  *
 868  * This function gets the result of UIC command completion
 869  *
 870  * Return: 0 on success; non-zero value on error.
 871  */
 872 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 873 {
 874         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 875                MASK_UIC_COMMAND_RESULT;
 876 }
 877
 878 /**
 879  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 880  * @hba: Pointer to adapter instance
 881  *
 882  * This function gets UIC command argument3
 883  *
 884  * Return: 0 on success; non-zero value on error.
 885  */
 886 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 887 {
 888         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 889 }
 890
 891 /**
 892  * ufshcd_get_req_rsp - returns the TR response transaction type
 893  * @ucd_rsp_ptr: pointer to response UPIU
 894  *
 895  * Return: UPIU type.
 896  */
 897 static inline enum upiu_response_transaction
 898 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 899 {
 900         return ucd_rsp_ptr->header.transaction_code;
 901 }
 902
 903 /**
 904  * ufshcd_is_exception_event - Check if the device raised an exception event
 905  * @ucd_rsp_ptr: pointer to response UPIU
 906  *
 907  * The function checks if the device raised an exception event indicated in
 908  * the Device Information field of response UPIU.
 909  *
 910  * Return: true if exception is raised, false otherwise.
 911  */
 912 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 913 {
 914         return ucd_rsp_ptr->header.device_information & 1;
 915 }
 916
 917 /**
 918  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 919  * @hba: per adapter instance
 920  */
 921 static inline void
 922 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 923 {
 924         ufshcd_writel(hba, INT_AGGR_ENABLE |
 925                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 926                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 927 }
 928
 929 /**
 930  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 931  * @hba: per adapter instance
 932  * @cnt: Interrupt aggregation counter threshold
 933  * @tmout: Interrupt aggregation timeout value
 934  */
 935 static inline void
 936 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 937 {
 938         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 939                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 940                       INT_AGGR_TIMEOUT_VAL(tmout),
 941                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 942 }
 943
 944 /**
 945  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 946  * @hba: per adapter instance
 947  */
 948 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 949 {
 950         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 951 }
 952
 953 /**
 954  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 955  *                      When run-stop registers are set to 1, it indicates the
 956  *                      host controller that it can process the requests
 957  * @hba: per adapter instance
 958  */
 959 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 960 {
 961         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 962                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 963         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 964                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 965 }
 966
 967 /**
 968  * ufshcd_hba_start - Start controller initialization sequence
 969  * @hba: per adapter instance
 970  */
 971 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 972 {
 973         u32 val = CONTROLLER_ENABLE;
 974
 975         if (ufshcd_crypto_enable(hba))
 976                 val |= CRYPTO_GENERAL_ENABLE;
 977
 978         ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
 979 }
 980
 981 /**
 982  * ufshcd_is_hba_active - Get controller state
 983  * @hba: per adapter instance
 984  *
 985  * Return: true if and only if the controller is active.
 986  */
 987 bool ufshcd_is_hba_active(struct ufs_hba *hba)
 988 {
 989         return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
 990 }
 991 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active);
 992
 993 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 994 {
 995         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 996         if (hba->ufs_version <= ufshci_version(1, 1))
 997                 return UFS_UNIPRO_VER_1_41;
 998         else
 999                 return UFS_UNIPRO_VER_1_6;
1000 }
1001 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
1002
1003 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
1004 {
1005         /*
1006          * If both host and device support UniPro ver1.6 or later, PA layer
1007          * parameters tuning happens during link startup itself.
1008          *
1009          * We can manually tune PA layer parameters if either host or device
1010          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
1011          * logic simple, we will only do manual tuning if local unipro version
1012          * doesn't support ver1.6 or later.
1013          */
1014         return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6;
1015 }
1016
1017 /**
1018  * ufshcd_set_clk_freq - set UFS controller clock frequencies
1019  * @hba: per adapter instance
1020  * @scale_up: If True, set max possible frequency othewise set low frequency
1021  *
1022  * Return: 0 if successful; < 0 upon failure.
1023  */
1024 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
1025 {
1026         int ret = 0;
1027         struct ufs_clk_info *clki;
1028         struct list_head *head = &hba->clk_list_head;
1029
1030         if (list_empty(head))
1031                 goto out;
1032
1033         list_for_each_entry(clki, head, list) {
1034                 if (!IS_ERR_OR_NULL(clki->clk)) {
1035                         if (scale_up && clki->max_freq) {
1036                                 if (clki->curr_freq == clki->max_freq)
1037                                         continue;
1038
1039                                 ret = clk_set_rate(clki->clk, clki->max_freq);
1040                                 if (ret) {
1041                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1042                                                 __func__, clki->name,
1043                                                 clki->max_freq, ret);
1044                                         break;
1045                                 }
1046                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1047                                                 "scaled up", clki->name,
1048                                                 clki->curr_freq,
1049                                                 clki->max_freq);
1050
1051                                 clki->curr_freq = clki->max_freq;
1052
1053                         } else if (!scale_up && clki->min_freq) {
1054                                 if (clki->curr_freq == clki->min_freq)
1055                                         continue;
1056
1057                                 ret = clk_set_rate(clki->clk, clki->min_freq);
1058                                 if (ret) {
1059                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1060                                                 __func__, clki->name,
1061                                                 clki->min_freq, ret);
1062                                         break;
1063                                 }
1064                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1065                                                 "scaled down", clki->name,
1066                                                 clki->curr_freq,
1067                                                 clki->min_freq);
1068                                 clki->curr_freq = clki->min_freq;
1069                         }
1070                 }
1071                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1072                                 clki->name, clk_get_rate(clki->clk));
1073         }
1074
1075 out:
1076         return ret;
1077 }
1078
1079 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table,
1080                            struct dev_pm_opp *opp, void *data,
1081                            bool scaling_down)
1082 {
1083         struct ufs_hba *hba = dev_get_drvdata(dev);
1084         struct list_head *head = &hba->clk_list_head;
1085         struct ufs_clk_info *clki;
1086         unsigned long freq;
1087         u8 idx = 0;
1088         int ret;
1089
1090         list_for_each_entry(clki, head, list) {
1091                 if (!IS_ERR_OR_NULL(clki->clk)) {
1092                         freq = dev_pm_opp_get_freq_indexed(opp, idx++);
1093
1094                         /* Do not set rate for clocks having frequency as 0 */
1095                         if (!freq)
1096                                 continue;
1097
1098                         ret = clk_set_rate(clki->clk, freq);
1099                         if (ret) {
1100                                 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n",
1101                                         __func__, clki->name, freq, ret);
1102                                 return ret;
1103                         }
1104
1105                         trace_ufshcd_clk_scaling(dev_name(dev),
1106                                 (scaling_down ? "scaled down" : "scaled up"),
1107                                 clki->name, hba->clk_scaling.target_freq, freq);
1108                 }
1109         }
1110
1111         return 0;
1112 }
1113 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks);
1114
1115 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq)
1116 {
1117         struct dev_pm_opp *opp;
1118         int ret;
1119
1120         opp = dev_pm_opp_find_freq_floor_indexed(hba->dev,
1121                                                  &freq, 0);
1122         if (IS_ERR(opp))
1123                 return PTR_ERR(opp);
1124
1125         ret = dev_pm_opp_set_opp(hba->dev, opp);
1126         dev_pm_opp_put(opp);
1127
1128         return ret;
1129 }
1130
1131 /**
1132  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1133  * @hba: per adapter instance
1134  * @freq: frequency to scale
1135  * @scale_up: True if scaling up and false if scaling down
1136  *
1137  * Return: 0 if successful; < 0 upon failure.
1138  */
1139 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
1140                              bool scale_up)
1141 {
1142         int ret = 0;
1143         ktime_t start = ktime_get();
1144
1145         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1146         if (ret)
1147                 goto out;
1148
1149         if (hba->use_pm_opp)
1150                 ret = ufshcd_opp_set_rate(hba, freq);
1151         else
1152                 ret = ufshcd_set_clk_freq(hba, scale_up);
1153         if (ret)
1154                 goto out;
1155
1156         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1157         if (ret) {
1158                 if (hba->use_pm_opp)
1159                         ufshcd_opp_set_rate(hba,
1160                                             hba->devfreq->previous_freq);
1161                 else
1162                         ufshcd_set_clk_freq(hba, !scale_up);
1163         }
1164
1165 out:
1166         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1167                         (scale_up ? "up" : "down"),
1168                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1169         return ret;
1170 }
1171
1172 /**
1173  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1174  * @hba: per adapter instance
1175  * @freq: frequency to scale
1176  * @scale_up: True if scaling up and false if scaling down
1177  *
1178  * Return: true if scaling is required, false otherwise.
1179  */
1180 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1181                                                unsigned long freq, bool scale_up)
1182 {
1183         struct ufs_clk_info *clki;
1184         struct list_head *head = &hba->clk_list_head;
1185
1186         if (list_empty(head))
1187                 return false;
1188
1189         if (hba->use_pm_opp)
1190                 return freq != hba->clk_scaling.target_freq;
1191
1192         list_for_each_entry(clki, head, list) {
1193                 if (!IS_ERR_OR_NULL(clki->clk)) {
1194                         if (scale_up && clki->max_freq) {
1195                                 if (clki->curr_freq == clki->max_freq)
1196                                         continue;
1197                                 return true;
1198                         } else if (!scale_up && clki->min_freq) {
1199                                 if (clki->curr_freq == clki->min_freq)
1200                                         continue;
1201                                 return true;
1202                         }
1203                 }
1204         }
1205
1206         return false;
1207 }
1208
1209 /*
1210  * Determine the number of pending commands by counting the bits in the SCSI
1211  * device budget maps. This approach has been selected because a bit is set in
1212  * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1213  * flag. The host_self_blocked flag can be modified by calling
1214  * scsi_block_requests() or scsi_unblock_requests().
1215  */
1216 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1217 {
1218         const struct scsi_device *sdev;
1219         u32 pending = 0;
1220
1221         lockdep_assert_held(hba->host->host_lock);
1222         __shost_for_each_device(sdev, hba->host)
1223                 pending += sbitmap_weight(&sdev->budget_map);
1224
1225         return pending;
1226 }
1227
1228 /*
1229  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1230  * has expired.
1231  *
1232  * Return: 0 upon success; -EBUSY upon timeout.
1233  */
1234 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1235                                         u64 wait_timeout_us)
1236 {
1237         unsigned long flags;
1238         int ret = 0;
1239         u32 tm_doorbell;
1240         u32 tr_pending;
1241         bool timeout = false, do_last_check = false;
1242         ktime_t start;
1243
1244         ufshcd_hold(hba);
1245         spin_lock_irqsave(hba->host->host_lock, flags);
1246         /*
1247          * Wait for all the outstanding tasks/transfer requests.
1248          * Verify by checking the doorbell registers are clear.
1249          */
1250         start = ktime_get();
1251         do {
1252                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1253                         ret = -EBUSY;
1254                         goto out;
1255                 }
1256
1257                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1258                 tr_pending = ufshcd_pending_cmds(hba);
1259                 if (!tm_doorbell && !tr_pending) {
1260                         timeout = false;
1261                         break;
1262                 } else if (do_last_check) {
1263                         break;
1264                 }
1265
1266                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1267                 io_schedule_timeout(msecs_to_jiffies(20));
1268                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1269                     wait_timeout_us) {
1270                         timeout = true;
1271                         /*
1272                          * We might have scheduled out for long time so make
1273                          * sure to check if doorbells are cleared by this time
1274                          * or not.
1275                          */
1276                         do_last_check = true;
1277                 }
1278                 spin_lock_irqsave(hba->host->host_lock, flags);
1279         } while (tm_doorbell || tr_pending);
1280
1281         if (timeout) {
1282                 dev_err(hba->dev,
1283                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1284                         __func__, tm_doorbell, tr_pending);
1285                 ret = -EBUSY;
1286         }
1287 out:
1288         spin_unlock_irqrestore(hba->host->host_lock, flags);
1289         ufshcd_release(hba);
1290         return ret;
1291 }
1292
1293 /**
1294  * ufshcd_scale_gear - scale up/down UFS gear
1295  * @hba: per adapter instance
1296  * @scale_up: True for scaling up gear and false for scaling down
1297  *
1298  * Return: 0 for success; -EBUSY if scaling can't happen at this time;
1299  * non-zero for any other errors.
1300  */
1301 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1302 {
1303         int ret = 0;
1304         struct ufs_pa_layer_attr new_pwr_info;
1305
1306         if (scale_up) {
1307                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
1308                        sizeof(struct ufs_pa_layer_attr));
1309         } else {
1310                 memcpy(&new_pwr_info, &hba->pwr_info,
1311                        sizeof(struct ufs_pa_layer_attr));
1312
1313                 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1314                     hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1315                         /* save the current power mode */
1316                         memcpy(&hba->clk_scaling.saved_pwr_info,
1317                                 &hba->pwr_info,
1318                                 sizeof(struct ufs_pa_layer_attr));
1319
1320                         /* scale down gear */
1321                         new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1322                         new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1323                 }
1324         }
1325
1326         /* check if the power mode needs to be changed or not? */
1327         ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1328         if (ret)
1329                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1330                         __func__, ret,
1331                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1332                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1333
1334         return ret;
1335 }
1336
1337 /*
1338  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1339  * has expired.
1340  *
1341  * Return: 0 upon success; -EBUSY upon timeout.
1342  */
1343 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
1344 {
1345         int ret = 0;
1346         /*
1347          * make sure that there are no outstanding requests when
1348          * clock scaling is in progress
1349          */
1350         ufshcd_scsi_block_requests(hba);
1351         mutex_lock(&hba->wb_mutex);
1352         down_write(&hba->clk_scaling_lock);
1353
1354         if (!hba->clk_scaling.is_allowed ||
1355             ufshcd_wait_for_doorbell_clr(hba, timeout_us)) {
1356                 ret = -EBUSY;
1357                 up_write(&hba->clk_scaling_lock);
1358                 mutex_unlock(&hba->wb_mutex);
1359                 ufshcd_scsi_unblock_requests(hba);
1360                 goto out;
1361         }
1362
1363         /* let's not get into low power until clock scaling is completed */
1364         ufshcd_hold(hba);
1365
1366 out:
1367         return ret;
1368 }
1369
1370 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up)
1371 {
1372         up_write(&hba->clk_scaling_lock);
1373
1374         /* Enable Write Booster if we have scaled up else disable it */
1375         if (ufshcd_enable_wb_if_scaling_up(hba) && !err)
1376                 ufshcd_wb_toggle(hba, scale_up);
1377
1378         mutex_unlock(&hba->wb_mutex);
1379
1380         ufshcd_scsi_unblock_requests(hba);
1381         ufshcd_release(hba);
1382 }
1383
1384 /**
1385  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1386  * @hba: per adapter instance
1387  * @freq: frequency to scale
1388  * @scale_up: True for scaling up and false for scalin down
1389  *
1390  * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero
1391  * for any other errors.
1392  */
1393 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq,
1394                                 bool scale_up)
1395 {
1396         int ret = 0;
1397
1398         ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC);
1399         if (ret)
1400                 return ret;
1401
1402         /* scale down the gear before scaling down clocks */
1403         if (!scale_up) {
1404                 ret = ufshcd_scale_gear(hba, false);
1405                 if (ret)
1406                         goto out_unprepare;
1407         }
1408
1409         ret = ufshcd_scale_clks(hba, freq, scale_up);
1410         if (ret) {
1411                 if (!scale_up)
1412                         ufshcd_scale_gear(hba, true);
1413                 goto out_unprepare;
1414         }
1415
1416         /* scale up the gear after scaling up clocks */
1417         if (scale_up) {
1418                 ret = ufshcd_scale_gear(hba, true);
1419                 if (ret) {
1420                         ufshcd_scale_clks(hba, hba->devfreq->previous_freq,
1421                                           false);
1422                         goto out_unprepare;
1423                 }
1424         }
1425
1426 out_unprepare:
1427         ufshcd_clock_scaling_unprepare(hba, ret, scale_up);
1428         return ret;
1429 }
1430
1431 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1432 {
1433         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1434                                            clk_scaling.suspend_work);
1435         unsigned long irq_flags;
1436
1437         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1438         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1439                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1440                 return;
1441         }
1442         hba->clk_scaling.is_suspended = true;
1443         hba->clk_scaling.window_start_t = 0;
1444         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1445
1446         devfreq_suspend_device(hba->devfreq);
1447 }
1448
1449 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1450 {
1451         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1452                                            clk_scaling.resume_work);
1453         unsigned long irq_flags;
1454
1455         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1456         if (!hba->clk_scaling.is_suspended) {
1457                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1458                 return;
1459         }
1460         hba->clk_scaling.is_suspended = false;
1461         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1462
1463         devfreq_resume_device(hba->devfreq);
1464 }
1465
1466 static int ufshcd_devfreq_target(struct device *dev,
1467                                 unsigned long *freq, u32 flags)
1468 {
1469         int ret = 0;
1470         struct ufs_hba *hba = dev_get_drvdata(dev);
1471         ktime_t start;
1472         bool scale_up = false, sched_clk_scaling_suspend_work = false;
1473         struct list_head *clk_list = &hba->clk_list_head;
1474         struct ufs_clk_info *clki;
1475         unsigned long irq_flags;
1476
1477         if (!ufshcd_is_clkscaling_supported(hba))
1478                 return -EINVAL;
1479
1480         if (hba->use_pm_opp) {
1481                 struct dev_pm_opp *opp;
1482
1483                 /* Get the recommended frequency from OPP framework */
1484                 opp = devfreq_recommended_opp(dev, freq, flags);
1485                 if (IS_ERR(opp))
1486                         return PTR_ERR(opp);
1487
1488                 dev_pm_opp_put(opp);
1489         } else {
1490                 /* Override with the closest supported frequency */
1491                 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info,
1492                                         list);
1493                 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1494         }
1495
1496         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1497         if (ufshcd_eh_in_progress(hba)) {
1498                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1499                 return 0;
1500         }
1501
1502         /* Skip scaling clock when clock scaling is suspended */
1503         if (hba->clk_scaling.is_suspended) {
1504                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1505                 dev_warn(hba->dev, "clock scaling is suspended, skip");
1506                 return 0;
1507         }
1508
1509         if (!hba->clk_scaling.active_reqs)
1510                 sched_clk_scaling_suspend_work = true;
1511
1512         if (list_empty(clk_list)) {
1513                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1514                 goto out;
1515         }
1516
1517         /* Decide based on the target or rounded-off frequency and update */
1518         if (hba->use_pm_opp)
1519                 scale_up = *freq > hba->clk_scaling.target_freq;
1520         else
1521                 scale_up = *freq == clki->max_freq;
1522
1523         if (!hba->use_pm_opp && !scale_up)
1524                 *freq = clki->min_freq;
1525
1526         /* Update the frequency */
1527         if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) {
1528                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1529                 ret = 0;
1530                 goto out; /* no state change required */
1531         }
1532         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1533
1534         start = ktime_get();
1535         ret = ufshcd_devfreq_scale(hba, *freq, scale_up);
1536         if (!ret)
1537                 hba->clk_scaling.target_freq = *freq;
1538
1539         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1540                 (scale_up ? "up" : "down"),
1541                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1542
1543 out:
1544         if (sched_clk_scaling_suspend_work && !scale_up)
1545                 queue_work(hba->clk_scaling.workq,
1546                            &hba->clk_scaling.suspend_work);
1547
1548         return ret;
1549 }
1550
1551 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1552                 struct devfreq_dev_status *stat)
1553 {
1554         struct ufs_hba *hba = dev_get_drvdata(dev);
1555         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1556         unsigned long flags;
1557         ktime_t curr_t;
1558
1559         if (!ufshcd_is_clkscaling_supported(hba))
1560                 return -EINVAL;
1561
1562         memset(stat, 0, sizeof(*stat));
1563
1564         spin_lock_irqsave(hba->host->host_lock, flags);
1565         curr_t = ktime_get();
1566         if (!scaling->window_start_t)
1567                 goto start_window;
1568
1569         /*
1570          * If current frequency is 0, then the ondemand governor considers
1571          * there's no initial frequency set. And it always requests to set
1572          * to max. frequency.
1573          */
1574         if (hba->use_pm_opp) {
1575                 stat->current_frequency = hba->clk_scaling.target_freq;
1576         } else {
1577                 struct list_head *clk_list = &hba->clk_list_head;
1578                 struct ufs_clk_info *clki;
1579
1580                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1581                 stat->current_frequency = clki->curr_freq;
1582         }
1583
1584         if (scaling->is_busy_started)
1585                 scaling->tot_busy_t += ktime_us_delta(curr_t,
1586                                 scaling->busy_start_t);
1587         stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1588         stat->busy_time = scaling->tot_busy_t;
1589 start_window:
1590         scaling->window_start_t = curr_t;
1591         scaling->tot_busy_t = 0;
1592
1593         if (scaling->active_reqs) {
1594                 scaling->busy_start_t = curr_t;
1595                 scaling->is_busy_started = true;
1596         } else {
1597                 scaling->busy_start_t = 0;
1598                 scaling->is_busy_started = false;
1599         }
1600         spin_unlock_irqrestore(hba->host->host_lock, flags);
1601         return 0;
1602 }
1603
1604 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1605 {
1606         struct list_head *clk_list = &hba->clk_list_head;
1607         struct ufs_clk_info *clki;
1608         struct devfreq *devfreq;
1609         int ret;
1610
1611         /* Skip devfreq if we don't have any clocks in the list */
1612         if (list_empty(clk_list))
1613                 return 0;
1614
1615         if (!hba->use_pm_opp) {
1616                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1617                 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1618                 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1619         }
1620
1621         ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1622                                          &hba->vps->ondemand_data);
1623         devfreq = devfreq_add_device(hba->dev,
1624                         &hba->vps->devfreq_profile,
1625                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1626                         &hba->vps->ondemand_data);
1627         if (IS_ERR(devfreq)) {
1628                 ret = PTR_ERR(devfreq);
1629                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1630
1631                 if (!hba->use_pm_opp) {
1632                         dev_pm_opp_remove(hba->dev, clki->min_freq);
1633                         dev_pm_opp_remove(hba->dev, clki->max_freq);
1634                 }
1635                 return ret;
1636         }
1637
1638         hba->devfreq = devfreq;
1639
1640         return 0;
1641 }
1642
1643 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1644 {
1645         struct list_head *clk_list = &hba->clk_list_head;
1646
1647         if (!hba->devfreq)
1648                 return;
1649
1650         devfreq_remove_device(hba->devfreq);
1651         hba->devfreq = NULL;
1652
1653         if (!hba->use_pm_opp) {
1654                 struct ufs_clk_info *clki;
1655
1656                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1657                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1658                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1659         }
1660 }
1661
1662 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1663 {
1664         unsigned long flags;
1665         bool suspend = false;
1666
1667         cancel_work_sync(&hba->clk_scaling.suspend_work);
1668         cancel_work_sync(&hba->clk_scaling.resume_work);
1669
1670         spin_lock_irqsave(hba->host->host_lock, flags);
1671         if (!hba->clk_scaling.is_suspended) {
1672                 suspend = true;
1673                 hba->clk_scaling.is_suspended = true;
1674                 hba->clk_scaling.window_start_t = 0;
1675         }
1676         spin_unlock_irqrestore(hba->host->host_lock, flags);
1677
1678         if (suspend)
1679                 devfreq_suspend_device(hba->devfreq);
1680 }
1681
1682 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1683 {
1684         unsigned long flags;
1685         bool resume = false;
1686
1687         spin_lock_irqsave(hba->host->host_lock, flags);
1688         if (hba->clk_scaling.is_suspended) {
1689                 resume = true;
1690                 hba->clk_scaling.is_suspended = false;
1691         }
1692         spin_unlock_irqrestore(hba->host->host_lock, flags);
1693
1694         if (resume)
1695                 devfreq_resume_device(hba->devfreq);
1696 }
1697
1698 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1699                 struct device_attribute *attr, char *buf)
1700 {
1701         struct ufs_hba *hba = dev_get_drvdata(dev);
1702
1703         return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1704 }
1705
1706 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1707                 struct device_attribute *attr, const char *buf, size_t count)
1708 {
1709         struct ufs_hba *hba = dev_get_drvdata(dev);
1710         u32 value;
1711         int err = 0;
1712
1713         if (kstrtou32(buf, 0, &value))
1714                 return -EINVAL;
1715
1716         down(&hba->host_sem);
1717         if (!ufshcd_is_user_access_allowed(hba)) {
1718                 err = -EBUSY;
1719                 goto out;
1720         }
1721
1722         value = !!value;
1723         if (value == hba->clk_scaling.is_enabled)
1724                 goto out;
1725
1726         ufshcd_rpm_get_sync(hba);
1727         ufshcd_hold(hba);
1728
1729         hba->clk_scaling.is_enabled = value;
1730
1731         if (value) {
1732                 ufshcd_resume_clkscaling(hba);
1733         } else {
1734                 ufshcd_suspend_clkscaling(hba);
1735                 err = ufshcd_devfreq_scale(hba, ULONG_MAX, true);
1736                 if (err)
1737                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1738                                         __func__, err);
1739         }
1740
1741         ufshcd_release(hba);
1742         ufshcd_rpm_put_sync(hba);
1743 out:
1744         up(&hba->host_sem);
1745         return err ? err : count;
1746 }
1747
1748 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1749 {
1750         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1751         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1752         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1753         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1754         hba->clk_scaling.enable_attr.attr.mode = 0644;
1755         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1756                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1757 }
1758
1759 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1760 {
1761         if (hba->clk_scaling.enable_attr.attr.name)
1762                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1763 }
1764
1765 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1766 {
1767         char wq_name[sizeof("ufs_clkscaling_00")];
1768
1769         if (!ufshcd_is_clkscaling_supported(hba))
1770                 return;
1771
1772         if (!hba->clk_scaling.min_gear)
1773                 hba->clk_scaling.min_gear = UFS_HS_G1;
1774
1775         INIT_WORK(&hba->clk_scaling.suspend_work,
1776                   ufshcd_clk_scaling_suspend_work);
1777         INIT_WORK(&hba->clk_scaling.resume_work,
1778                   ufshcd_clk_scaling_resume_work);
1779
1780         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1781                  hba->host->host_no);
1782         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1783
1784         hba->clk_scaling.is_initialized = true;
1785 }
1786
1787 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1788 {
1789         if (!hba->clk_scaling.is_initialized)
1790                 return;
1791
1792         ufshcd_remove_clk_scaling_sysfs(hba);
1793         destroy_workqueue(hba->clk_scaling.workq);
1794         ufshcd_devfreq_remove(hba);
1795         hba->clk_scaling.is_initialized = false;
1796 }
1797
1798 static void ufshcd_ungate_work(struct work_struct *work)
1799 {
1800         int ret;
1801         unsigned long flags;
1802         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1803                         clk_gating.ungate_work);
1804
1805         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1806
1807         spin_lock_irqsave(hba->host->host_lock, flags);
1808         if (hba->clk_gating.state == CLKS_ON) {
1809                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1810                 return;
1811         }
1812
1813         spin_unlock_irqrestore(hba->host->host_lock, flags);
1814         ufshcd_hba_vreg_set_hpm(hba);
1815         ufshcd_setup_clocks(hba, true);
1816
1817         ufshcd_enable_irq(hba);
1818
1819         /* Exit from hibern8 */
1820         if (ufshcd_can_hibern8_during_gating(hba)) {
1821                 /* Prevent gating in this path */
1822                 hba->clk_gating.is_suspended = true;
1823                 if (ufshcd_is_link_hibern8(hba)) {
1824                         ret = ufshcd_uic_hibern8_exit(hba);
1825                         if (ret)
1826                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1827                                         __func__, ret);
1828                         else
1829                                 ufshcd_set_link_active(hba);
1830                 }
1831                 hba->clk_gating.is_suspended = false;
1832         }
1833 }
1834
1835 /**
1836  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1837  * Also, exit from hibern8 mode and set the link as active.
1838  * @hba: per adapter instance
1839  */
1840 void ufshcd_hold(struct ufs_hba *hba)
1841 {
1842         bool flush_result;
1843         unsigned long flags;
1844
1845         if (!ufshcd_is_clkgating_allowed(hba) ||
1846             !hba->clk_gating.is_initialized)
1847                 return;
1848         spin_lock_irqsave(hba->host->host_lock, flags);
1849         hba->clk_gating.active_reqs++;
1850
1851 start:
1852         switch (hba->clk_gating.state) {
1853         case CLKS_ON:
1854                 /*
1855                  * Wait for the ungate work to complete if in progress.
1856                  * Though the clocks may be in ON state, the link could
1857                  * still be in hibner8 state if hibern8 is allowed
1858                  * during clock gating.
1859                  * Make sure we exit hibern8 state also in addition to
1860                  * clocks being ON.
1861                  */
1862                 if (ufshcd_can_hibern8_during_gating(hba) &&
1863                     ufshcd_is_link_hibern8(hba)) {
1864                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1865                         flush_result = flush_work(&hba->clk_gating.ungate_work);
1866                         if (hba->clk_gating.is_suspended && !flush_result)
1867                                 return;
1868                         spin_lock_irqsave(hba->host->host_lock, flags);
1869                         goto start;
1870                 }
1871                 break;
1872         case REQ_CLKS_OFF:
1873                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1874                         hba->clk_gating.state = CLKS_ON;
1875                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1876                                                 hba->clk_gating.state);
1877                         break;
1878                 }
1879                 /*
1880                  * If we are here, it means gating work is either done or
1881                  * currently running. Hence, fall through to cancel gating
1882                  * work and to enable clocks.
1883                  */
1884                 fallthrough;
1885         case CLKS_OFF:
1886                 hba->clk_gating.state = REQ_CLKS_ON;
1887                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1888                                         hba->clk_gating.state);
1889                 queue_work(hba->clk_gating.clk_gating_workq,
1890                            &hba->clk_gating.ungate_work);
1891                 /*
1892                  * fall through to check if we should wait for this
1893                  * work to be done or not.
1894                  */
1895                 fallthrough;
1896         case REQ_CLKS_ON:
1897                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1898                 flush_work(&hba->clk_gating.ungate_work);
1899                 /* Make sure state is CLKS_ON before returning */
1900                 spin_lock_irqsave(hba->host->host_lock, flags);
1901                 goto start;
1902         default:
1903                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1904                                 __func__, hba->clk_gating.state);
1905                 break;
1906         }
1907         spin_unlock_irqrestore(hba->host->host_lock, flags);
1908 }
1909 EXPORT_SYMBOL_GPL(ufshcd_hold);
1910
1911 static void ufshcd_gate_work(struct work_struct *work)
1912 {
1913         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1914                         clk_gating.gate_work.work);
1915         unsigned long flags;
1916         int ret;
1917
1918         spin_lock_irqsave(hba->host->host_lock, flags);
1919         /*
1920          * In case you are here to cancel this work the gating state
1921          * would be marked as REQ_CLKS_ON. In this case save time by
1922          * skipping the gating work and exit after changing the clock
1923          * state to CLKS_ON.
1924          */
1925         if (hba->clk_gating.is_suspended ||
1926                 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1927                 hba->clk_gating.state = CLKS_ON;
1928                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1929                                         hba->clk_gating.state);
1930                 goto rel_lock;
1931         }
1932
1933         if (ufshcd_is_ufs_dev_busy(hba) || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1934                 goto rel_lock;
1935
1936         spin_unlock_irqrestore(hba->host->host_lock, flags);
1937
1938         /* put the link into hibern8 mode before turning off clocks */
1939         if (ufshcd_can_hibern8_during_gating(hba)) {
1940                 ret = ufshcd_uic_hibern8_enter(hba);
1941                 if (ret) {
1942                         hba->clk_gating.state = CLKS_ON;
1943                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1944                                         __func__, ret);
1945                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1946                                                 hba->clk_gating.state);
1947                         goto out;
1948                 }
1949                 ufshcd_set_link_hibern8(hba);
1950         }
1951
1952         ufshcd_disable_irq(hba);
1953
1954         ufshcd_setup_clocks(hba, false);
1955
1956         /* Put the host controller in low power mode if possible */
1957         ufshcd_hba_vreg_set_lpm(hba);
1958         /*
1959          * In case you are here to cancel this work the gating state
1960          * would be marked as REQ_CLKS_ON. In this case keep the state
1961          * as REQ_CLKS_ON which would anyway imply that clocks are off
1962          * and a request to turn them on is pending. By doing this way,
1963          * we keep the state machine in tact and this would ultimately
1964          * prevent from doing cancel work multiple times when there are
1965          * new requests arriving before the current cancel work is done.
1966          */
1967         spin_lock_irqsave(hba->host->host_lock, flags);
1968         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1969                 hba->clk_gating.state = CLKS_OFF;
1970                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1971                                         hba->clk_gating.state);
1972         }
1973 rel_lock:
1974         spin_unlock_irqrestore(hba->host->host_lock, flags);
1975 out:
1976         return;
1977 }
1978
1979 /* host lock must be held before calling this variant */
1980 static void __ufshcd_release(struct ufs_hba *hba)
1981 {
1982         if (!ufshcd_is_clkgating_allowed(hba))
1983                 return;
1984
1985         hba->clk_gating.active_reqs--;
1986
1987         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1988             hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1989             hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
1990             hba->active_uic_cmd || hba->uic_async_done ||
1991             hba->clk_gating.state == CLKS_OFF)
1992                 return;
1993
1994         hba->clk_gating.state = REQ_CLKS_OFF;
1995         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1996         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1997                            &hba->clk_gating.gate_work,
1998                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1999 }
2000
2001 void ufshcd_release(struct ufs_hba *hba)
2002 {
2003         unsigned long flags;
2004
2005         spin_lock_irqsave(hba->host->host_lock, flags);
2006         __ufshcd_release(hba);
2007         spin_unlock_irqrestore(hba->host->host_lock, flags);
2008 }
2009 EXPORT_SYMBOL_GPL(ufshcd_release);
2010
2011 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
2012                 struct device_attribute *attr, char *buf)
2013 {
2014         struct ufs_hba *hba = dev_get_drvdata(dev);
2015
2016         return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
2017 }
2018
2019 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
2020 {
2021         struct ufs_hba *hba = dev_get_drvdata(dev);
2022         unsigned long flags;
2023
2024         spin_lock_irqsave(hba->host->host_lock, flags);
2025         hba->clk_gating.delay_ms = value;
2026         spin_unlock_irqrestore(hba->host->host_lock, flags);
2027 }
2028 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
2029
2030 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
2031                 struct device_attribute *attr, const char *buf, size_t count)
2032 {
2033         unsigned long value;
2034
2035         if (kstrtoul(buf, 0, &value))
2036                 return -EINVAL;
2037
2038         ufshcd_clkgate_delay_set(dev, value);
2039         return count;
2040 }
2041
2042 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
2043                 struct device_attribute *attr, char *buf)
2044 {
2045         struct ufs_hba *hba = dev_get_drvdata(dev);
2046
2047         return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
2048 }
2049
2050 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
2051                 struct device_attribute *attr, const char *buf, size_t count)
2052 {
2053         struct ufs_hba *hba = dev_get_drvdata(dev);
2054         unsigned long flags;
2055         u32 value;
2056
2057         if (kstrtou32(buf, 0, &value))
2058                 return -EINVAL;
2059
2060         value = !!value;
2061
2062         spin_lock_irqsave(hba->host->host_lock, flags);
2063         if (value == hba->clk_gating.is_enabled)
2064                 goto out;
2065
2066         if (value)
2067                 __ufshcd_release(hba);
2068         else
2069                 hba->clk_gating.active_reqs++;
2070
2071         hba->clk_gating.is_enabled = value;
2072 out:
2073         spin_unlock_irqrestore(hba->host->host_lock, flags);
2074         return count;
2075 }
2076
2077 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
2078 {
2079         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
2080         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
2081         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
2082         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
2083         hba->clk_gating.delay_attr.attr.mode = 0644;
2084         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
2085                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
2086
2087         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
2088         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
2089         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
2090         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
2091         hba->clk_gating.enable_attr.attr.mode = 0644;
2092         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
2093                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
2094 }
2095
2096 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
2097 {
2098         if (hba->clk_gating.delay_attr.attr.name)
2099                 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
2100         if (hba->clk_gating.enable_attr.attr.name)
2101                 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
2102 }
2103
2104 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
2105 {
2106         char wq_name[sizeof("ufs_clk_gating_00")];
2107
2108         if (!ufshcd_is_clkgating_allowed(hba))
2109                 return;
2110
2111         hba->clk_gating.state = CLKS_ON;
2112
2113         hba->clk_gating.delay_ms = 150;
2114         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
2115         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
2116
2117         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
2118                  hba->host->host_no);
2119         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
2120                                         WQ_MEM_RECLAIM | WQ_HIGHPRI);
2121
2122         ufshcd_init_clk_gating_sysfs(hba);
2123
2124         hba->clk_gating.is_enabled = true;
2125         hba->clk_gating.is_initialized = true;
2126 }
2127
2128 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2129 {
2130         if (!hba->clk_gating.is_initialized)
2131                 return;
2132
2133         ufshcd_remove_clk_gating_sysfs(hba);
2134
2135         /* Ungate the clock if necessary. */
2136         ufshcd_hold(hba);
2137         hba->clk_gating.is_initialized = false;
2138         ufshcd_release(hba);
2139
2140         destroy_workqueue(hba->clk_gating.clk_gating_workq);
2141 }
2142
2143 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2144 {
2145         bool queue_resume_work = false;
2146         ktime_t curr_t = ktime_get();
2147         unsigned long flags;
2148
2149         if (!ufshcd_is_clkscaling_supported(hba))
2150                 return;
2151
2152         spin_lock_irqsave(hba->host->host_lock, flags);
2153         if (!hba->clk_scaling.active_reqs++)
2154                 queue_resume_work = true;
2155
2156         if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2157                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2158                 return;
2159         }
2160
2161         if (queue_resume_work)
2162                 queue_work(hba->clk_scaling.workq,
2163                            &hba->clk_scaling.resume_work);
2164
2165         if (!hba->clk_scaling.window_start_t) {
2166                 hba->clk_scaling.window_start_t = curr_t;
2167                 hba->clk_scaling.tot_busy_t = 0;
2168                 hba->clk_scaling.is_busy_started = false;
2169         }
2170
2171         if (!hba->clk_scaling.is_busy_started) {
2172                 hba->clk_scaling.busy_start_t = curr_t;
2173                 hba->clk_scaling.is_busy_started = true;
2174         }
2175         spin_unlock_irqrestore(hba->host->host_lock, flags);
2176 }
2177
2178 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2179 {
2180         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2181         unsigned long flags;
2182
2183         if (!ufshcd_is_clkscaling_supported(hba))
2184                 return;
2185
2186         spin_lock_irqsave(hba->host->host_lock, flags);
2187         hba->clk_scaling.active_reqs--;
2188         if (!scaling->active_reqs && scaling->is_busy_started) {
2189                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2190                                         scaling->busy_start_t));
2191                 scaling->busy_start_t = 0;
2192                 scaling->is_busy_started = false;
2193         }
2194         spin_unlock_irqrestore(hba->host->host_lock, flags);
2195 }
2196
2197 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2198 {
2199         if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2200                 return READ;
2201         else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2202                 return WRITE;
2203         else
2204                 return -EINVAL;
2205 }
2206
2207 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2208                                                 struct ufshcd_lrb *lrbp)
2209 {
2210         const struct ufs_hba_monitor *m = &hba->monitor;
2211
2212         return (m->enabled && lrbp && lrbp->cmd &&
2213                 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2214                 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2215 }
2216
2217 static void ufshcd_start_monitor(struct ufs_hba *hba,
2218                                  const struct ufshcd_lrb *lrbp)
2219 {
2220         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2221         unsigned long flags;
2222
2223         spin_lock_irqsave(hba->host->host_lock, flags);
2224         if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2225                 hba->monitor.busy_start_ts[dir] = ktime_get();
2226         spin_unlock_irqrestore(hba->host->host_lock, flags);
2227 }
2228
2229 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
2230 {
2231         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2232         unsigned long flags;
2233
2234         spin_lock_irqsave(hba->host->host_lock, flags);
2235         if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2236                 const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2237                 struct ufs_hba_monitor *m = &hba->monitor;
2238                 ktime_t now, inc, lat;
2239
2240                 now = lrbp->compl_time_stamp;
2241                 inc = ktime_sub(now, m->busy_start_ts[dir]);
2242                 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2243                 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2244
2245                 /* Update latencies */
2246                 m->nr_req[dir]++;
2247                 lat = ktime_sub(now, lrbp->issue_time_stamp);
2248                 m->lat_sum[dir] += lat;
2249                 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2250                         m->lat_max[dir] = lat;
2251                 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2252                         m->lat_min[dir] = lat;
2253
2254                 m->nr_queued[dir]--;
2255                 /* Push forward the busy start of monitor */
2256                 m->busy_start_ts[dir] = now;
2257         }
2258         spin_unlock_irqrestore(hba->host->host_lock, flags);
2259 }
2260
2261 /**
2262  * ufshcd_send_command - Send SCSI or device management commands
2263  * @hba: per adapter instance
2264  * @task_tag: Task tag of the command
2265  * @hwq: pointer to hardware queue instance
2266  */
2267 static inline
2268 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
2269                          struct ufs_hw_queue *hwq)
2270 {
2271         struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2272         unsigned long flags;
2273
2274         lrbp->issue_time_stamp = ktime_get();
2275         lrbp->issue_time_stamp_local_clock = local_clock();
2276         lrbp->compl_time_stamp = ktime_set(0, 0);
2277         lrbp->compl_time_stamp_local_clock = 0;
2278         ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2279         if (lrbp->cmd)
2280                 ufshcd_clk_scaling_start_busy(hba);
2281         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2282                 ufshcd_start_monitor(hba, lrbp);
2283
2284         if (is_mcq_enabled(hba)) {
2285                 int utrd_size = sizeof(struct utp_transfer_req_desc);
2286                 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
2287                 struct utp_transfer_req_desc *dest;
2288
2289                 spin_lock(&hwq->sq_lock);
2290                 dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
2291                 memcpy(dest, src, utrd_size);
2292                 ufshcd_inc_sq_tail(hwq);
2293                 spin_unlock(&hwq->sq_lock);
2294         } else {
2295                 spin_lock_irqsave(&hba->outstanding_lock, flags);
2296                 if (hba->vops && hba->vops->setup_xfer_req)
2297                         hba->vops->setup_xfer_req(hba, lrbp->task_tag,
2298                                                   !!lrbp->cmd);
2299                 __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
2300                 ufshcd_writel(hba, 1 << lrbp->task_tag,
2301                               REG_UTP_TRANSFER_REQ_DOOR_BELL);
2302                 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2303         }
2304 }
2305
2306 /**
2307  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2308  * @lrbp: pointer to local reference block
2309  */
2310 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2311 {
2312         u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2313         u16 resp_len;
2314         int len;
2315
2316         resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length);
2317         if (sense_buffer && resp_len) {
2318                 int len_to_copy;
2319
2320                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2321                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2322
2323                 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2324                        len_to_copy);
2325         }
2326 }
2327
2328 /**
2329  * ufshcd_copy_query_response() - Copy the Query Response and the data
2330  * descriptor
2331  * @hba: per adapter instance
2332  * @lrbp: pointer to local reference block
2333  *
2334  * Return: 0 upon success; < 0 upon failure.
2335  */
2336 static
2337 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2338 {
2339         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2340
2341         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2342
2343         /* Get the descriptor */
2344         if (hba->dev_cmd.query.descriptor &&
2345             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2346                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2347                                 GENERAL_UPIU_REQUEST_SIZE;
2348                 u16 resp_len;
2349                 u16 buf_len;
2350
2351                 /* data segment length */
2352                 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
2353                                        .data_segment_length);
2354                 buf_len = be16_to_cpu(
2355                                 hba->dev_cmd.query.request.upiu_req.length);
2356                 if (likely(buf_len >= resp_len)) {
2357                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2358                 } else {
2359                         dev_warn(hba->dev,
2360                                  "%s: rsp size %d is bigger than buffer size %d",
2361                                  __func__, resp_len, buf_len);
2362                         return -EINVAL;
2363                 }
2364         }
2365
2366         return 0;
2367 }
2368
2369 /**
2370  * ufshcd_hba_capabilities - Read controller capabilities
2371  * @hba: per adapter instance
2372  *
2373  * Return: 0 on success, negative on error.
2374  */
2375 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2376 {
2377         int err;
2378
2379         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2380         if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS)
2381                 hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT;
2382
2383         /* nutrs and nutmrs are 0 based values */
2384         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2385         hba->nutmrs =
2386         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2387         hba->reserved_slot = hba->nutrs - 1;
2388
2389         /* Read crypto capabilities */
2390         err = ufshcd_hba_init_crypto_capabilities(hba);
2391         if (err) {
2392                 dev_err(hba->dev, "crypto setup failed\n");
2393                 return err;
2394         }
2395
2396         hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
2397         if (!hba->mcq_sup)
2398                 return 0;
2399
2400         hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
2401         hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
2402                                      hba->mcq_capabilities);
2403
2404         return 0;
2405 }
2406
2407 /**
2408  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2409  *                            to accept UIC commands
2410  * @hba: per adapter instance
2411  *
2412  * Return: true on success, else false.
2413  */
2414 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2415 {
2416         u32 val;
2417         int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY,
2418                                     500, UIC_CMD_TIMEOUT * 1000, false, hba,
2419                                     REG_CONTROLLER_STATUS);
2420         return ret == 0;
2421 }
2422
2423 /**
2424  * ufshcd_get_upmcrs - Get the power mode change request status
2425  * @hba: Pointer to adapter instance
2426  *
2427  * This function gets the UPMCRS field of HCS register
2428  *
2429  * Return: value of UPMCRS field.
2430  */
2431 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2432 {
2433         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2434 }
2435
2436 /**
2437  * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2438  * @hba: per adapter instance
2439  * @uic_cmd: UIC command
2440  */
2441 static inline void
2442 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2443 {
2444         lockdep_assert_held(&hba->uic_cmd_mutex);
2445
2446         WARN_ON(hba->active_uic_cmd);
2447
2448         hba->active_uic_cmd = uic_cmd;
2449
2450         /* Write Args */
2451         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2452         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2453         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2454
2455         ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2456
2457         /* Write UIC Cmd */
2458         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2459                       REG_UIC_COMMAND);
2460 }
2461
2462 /**
2463  * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2464  * @hba: per adapter instance
2465  * @uic_cmd: UIC command
2466  *
2467  * Return: 0 only if success.
2468  */
2469 static int
2470 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2471 {
2472         int ret;
2473         unsigned long flags;
2474
2475         lockdep_assert_held(&hba->uic_cmd_mutex);
2476
2477         if (wait_for_completion_timeout(&uic_cmd->done,
2478                                         msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2479                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2480         } else {
2481                 ret = -ETIMEDOUT;
2482                 dev_err(hba->dev,
2483                         "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2484                         uic_cmd->command, uic_cmd->argument3);
2485
2486                 if (!uic_cmd->cmd_active) {
2487                         dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2488                                 __func__);
2489                         ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2490                 }
2491         }
2492
2493         spin_lock_irqsave(hba->host->host_lock, flags);
2494         hba->active_uic_cmd = NULL;
2495         spin_unlock_irqrestore(hba->host->host_lock, flags);
2496
2497         return ret;
2498 }
2499
2500 /**
2501  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2502  * @hba: per adapter instance
2503  * @uic_cmd: UIC command
2504  * @completion: initialize the completion only if this is set to true
2505  *
2506  * Return: 0 only if success.
2507  */
2508 static int
2509 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2510                       bool completion)
2511 {
2512         lockdep_assert_held(&hba->uic_cmd_mutex);
2513
2514         if (!ufshcd_ready_for_uic_cmd(hba)) {
2515                 dev_err(hba->dev,
2516                         "Controller not ready to accept UIC commands\n");
2517                 return -EIO;
2518         }
2519
2520         if (completion)
2521                 init_completion(&uic_cmd->done);
2522
2523         uic_cmd->cmd_active = 1;
2524         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2525
2526         return 0;
2527 }
2528
2529 /**
2530  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2531  * @hba: per adapter instance
2532  * @uic_cmd: UIC command
2533  *
2534  * Return: 0 only if success.
2535  */
2536 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2537 {
2538         int ret;
2539
2540         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2541                 return 0;
2542
2543         ufshcd_hold(hba);
2544         mutex_lock(&hba->uic_cmd_mutex);
2545         ufshcd_add_delay_before_dme_cmd(hba);
2546
2547         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2548         if (!ret)
2549                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2550
2551         mutex_unlock(&hba->uic_cmd_mutex);
2552
2553         ufshcd_release(hba);
2554         return ret;
2555 }
2556
2557 /**
2558  * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format)
2559  * @hba:        per-adapter instance
2560  * @lrbp:       pointer to local reference block
2561  * @sg_entries: The number of sg lists actually used
2562  * @sg_list:    Pointer to SG list
2563  */
2564 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries,
2565                                struct scatterlist *sg_list)
2566 {
2567         struct ufshcd_sg_entry *prd;
2568         struct scatterlist *sg;
2569         int i;
2570
2571         if (sg_entries) {
2572
2573                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2574                         lrbp->utr_descriptor_ptr->prd_table_length =
2575                                 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba));
2576                 else
2577                         lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries);
2578
2579                 prd = lrbp->ucd_prdt_ptr;
2580
2581                 for_each_sg(sg_list, sg, sg_entries, i) {
2582                         const unsigned int len = sg_dma_len(sg);
2583
2584                         /*
2585                          * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2586                          * based value that indicates the length, in bytes, of
2587                          * the data block. A maximum of length of 256KB may
2588                          * exist for any entry. Bits 1:0 of this field shall be
2589                          * 11b to indicate Dword granularity. A value of '3'
2590                          * indicates 4 bytes, '7' indicates 8 bytes, etc."
2591                          */
2592                         WARN_ONCE(len > SZ_256K, "len = %#x\n", len);
2593                         prd->size = cpu_to_le32(len - 1);
2594                         prd->addr = cpu_to_le64(sg->dma_address);
2595                         prd->reserved = 0;
2596                         prd = (void *)prd + ufshcd_sg_entry_size(hba);
2597                 }
2598         } else {
2599                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2600         }
2601 }
2602
2603 /**
2604  * ufshcd_map_sg - Map scatter-gather list to prdt
2605  * @hba: per adapter instance
2606  * @lrbp: pointer to local reference block
2607  *
2608  * Return: 0 in case of success, non-zero value in case of failure.
2609  */
2610 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2611 {
2612         struct scsi_cmnd *cmd = lrbp->cmd;
2613         int sg_segments = scsi_dma_map(cmd);
2614
2615         if (sg_segments < 0)
2616                 return sg_segments;
2617
2618         ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
2619
2620         return 0;
2621 }
2622
2623 /**
2624  * ufshcd_enable_intr - enable interrupts
2625  * @hba: per adapter instance
2626  * @intrs: interrupt bits
2627  */
2628 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2629 {
2630         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2631
2632         if (hba->ufs_version == ufshci_version(1, 0)) {
2633                 u32 rw;
2634                 rw = set & INTERRUPT_MASK_RW_VER_10;
2635                 set = rw | ((set ^ intrs) & intrs);
2636         } else {
2637                 set |= intrs;
2638         }
2639
2640         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2641 }
2642
2643 /**
2644  * ufshcd_disable_intr - disable interrupts
2645  * @hba: per adapter instance
2646  * @intrs: interrupt bits
2647  */
2648 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2649 {
2650         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2651
2652         if (hba->ufs_version == ufshci_version(1, 0)) {
2653                 u32 rw;
2654                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2655                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2656                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2657
2658         } else {
2659                 set &= ~intrs;
2660         }
2661
2662         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2663 }
2664
2665 /**
2666  * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request
2667  * descriptor according to request
2668  * @lrbp: pointer to local reference block
2669  * @upiu_flags: flags required in the header
2670  * @cmd_dir: requests data direction
2671  * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments)
2672  */
2673 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags,
2674                                         enum dma_data_direction cmd_dir, int ehs_length)
2675 {
2676         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2677         struct request_desc_header *h = &req_desc->header;
2678         enum utp_data_direction data_direction;
2679
2680         *h = (typeof(*h)){ };
2681
2682         if (cmd_dir == DMA_FROM_DEVICE) {
2683                 data_direction = UTP_DEVICE_TO_HOST;
2684                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2685         } else if (cmd_dir == DMA_TO_DEVICE) {
2686                 data_direction = UTP_HOST_TO_DEVICE;
2687                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2688         } else {
2689                 data_direction = UTP_NO_DATA_TRANSFER;
2690                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2691         }
2692
2693         h->command_type = lrbp->command_type;
2694         h->data_direction = data_direction;
2695         h->ehs_length = ehs_length;
2696
2697         if (lrbp->intr_cmd)
2698                 h->interrupt = 1;
2699
2700         /* Prepare crypto related dwords */
2701         ufshcd_prepare_req_desc_hdr_crypto(lrbp, h);
2702
2703         /*
2704          * assigning invalid value for command status. Controller
2705          * updates OCS on command completion, with the command
2706          * status
2707          */
2708         h->ocs = OCS_INVALID_COMMAND_STATUS;
2709
2710         req_desc->prd_table_length = 0;
2711 }
2712
2713 /**
2714  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2715  * for scsi commands
2716  * @lrbp: local reference block pointer
2717  * @upiu_flags: flags
2718  */
2719 static
2720 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2721 {
2722         struct scsi_cmnd *cmd = lrbp->cmd;
2723         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2724         unsigned short cdb_len;
2725
2726         ucd_req_ptr->header = (struct utp_upiu_header){
2727                 .transaction_code = UPIU_TRANSACTION_COMMAND,
2728                 .flags = upiu_flags,
2729                 .lun = lrbp->lun,
2730                 .task_tag = lrbp->task_tag,
2731                 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI,
2732         };
2733
2734         WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag);
2735
2736         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2737
2738         cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2739         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2740         memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2741
2742         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2743 }
2744
2745 /**
2746  * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
2747  * @hba: UFS hba
2748  * @lrbp: local reference block pointer
2749  * @upiu_flags: flags
2750  */
2751 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2752                                 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2753 {
2754         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2755         struct ufs_query *query = &hba->dev_cmd.query;
2756         u16 len = be16_to_cpu(query->request.upiu_req.length);
2757
2758         /* Query request header */
2759         ucd_req_ptr->header = (struct utp_upiu_header){
2760                 .transaction_code = UPIU_TRANSACTION_QUERY_REQ,
2761                 .flags = upiu_flags,
2762                 .lun = lrbp->lun,
2763                 .task_tag = lrbp->task_tag,
2764                 .query_function = query->request.query_func,
2765                 /* Data segment length only need for WRITE_DESC */
2766                 .data_segment_length =
2767                         query->request.upiu_req.opcode ==
2768                                         UPIU_QUERY_OPCODE_WRITE_DESC ?
2769                                 cpu_to_be16(len) :
2770                                 0,
2771         };
2772
2773         /* Copy the Query Request buffer as is */
2774         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2775                         QUERY_OSF_SIZE);
2776
2777         /* Copy the Descriptor */
2778         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2779                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2780
2781         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2782 }
2783
2784 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2785 {
2786         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2787
2788         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2789
2790         ucd_req_ptr->header = (struct utp_upiu_header){
2791                 .transaction_code = UPIU_TRANSACTION_NOP_OUT,
2792                 .task_tag = lrbp->task_tag,
2793         };
2794
2795         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2796 }
2797
2798 /**
2799  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2800  *                           for Device Management Purposes
2801  * @hba: per adapter instance
2802  * @lrbp: pointer to local reference block
2803  *
2804  * Return: 0 upon success; < 0 upon failure.
2805  */
2806 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2807                                       struct ufshcd_lrb *lrbp)
2808 {
2809         u8 upiu_flags;
2810         int ret = 0;
2811
2812         if (hba->ufs_version <= ufshci_version(1, 1))
2813                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2814         else
2815                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2816
2817         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0);
2818         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2819                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2820         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2821                 ufshcd_prepare_utp_nop_upiu(lrbp);
2822         else
2823                 ret = -EINVAL;
2824
2825         return ret;
2826 }
2827
2828 /**
2829  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2830  *                         for SCSI Purposes
2831  * @hba: per adapter instance
2832  * @lrbp: pointer to local reference block
2833  */
2834 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2835 {
2836         struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
2837         unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
2838         u8 upiu_flags;
2839
2840         if (hba->ufs_version <= ufshci_version(1, 1))
2841                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2842         else
2843                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2844
2845         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2846                                     lrbp->cmd->sc_data_direction, 0);
2847         if (ioprio_class == IOPRIO_CLASS_RT)
2848                 upiu_flags |= UPIU_CMD_FLAGS_CP;
2849         ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2850 }
2851
2852 /**
2853  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2854  * @upiu_wlun_id: UPIU W-LUN id
2855  *
2856  * Return: SCSI W-LUN id.
2857  */
2858 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2859 {
2860         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2861 }
2862
2863 static inline bool is_device_wlun(struct scsi_device *sdev)
2864 {
2865         return sdev->lun ==
2866                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2867 }
2868
2869 /*
2870  * Associate the UFS controller queue with the default and poll HCTX types.
2871  * Initialize the mq_map[] arrays.
2872  */
2873 static void ufshcd_map_queues(struct Scsi_Host *shost)
2874 {
2875         struct ufs_hba *hba = shost_priv(shost);
2876         int i, queue_offset = 0;
2877
2878         if (!is_mcq_supported(hba)) {
2879                 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1;
2880                 hba->nr_queues[HCTX_TYPE_READ] = 0;
2881                 hba->nr_queues[HCTX_TYPE_POLL] = 1;
2882                 hba->nr_hw_queues = 1;
2883         }
2884
2885         for (i = 0; i < shost->nr_maps; i++) {
2886                 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2887
2888                 map->nr_queues = hba->nr_queues[i];
2889                 if (!map->nr_queues)
2890                         continue;
2891                 map->queue_offset = queue_offset;
2892                 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba))
2893                         map->queue_offset = 0;
2894
2895                 blk_mq_map_queues(map);
2896                 queue_offset += map->nr_queues;
2897         }
2898 }
2899
2900 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2901 {
2902         struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2903                 i * ufshcd_get_ucd_size(hba);
2904         struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2905         dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2906                 i * ufshcd_get_ucd_size(hba);
2907         u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2908                                        response_upiu);
2909         u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2910
2911         lrb->utr_descriptor_ptr = utrdlp + i;
2912         lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2913                 i * sizeof(struct utp_transfer_req_desc);
2914         lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
2915         lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2916         lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2917         lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2918         lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2919         lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2920 }
2921
2922 /**
2923  * ufshcd_queuecommand - main entry point for SCSI requests
2924  * @host: SCSI host pointer
2925  * @cmd: command from SCSI Midlayer
2926  *
2927  * Return: 0 for success, non-zero in case of failure.
2928  */
2929 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2930 {
2931         struct ufs_hba *hba = shost_priv(host);
2932         int tag = scsi_cmd_to_rq(cmd)->tag;
2933         struct ufshcd_lrb *lrbp;
2934         int err = 0;
2935         struct ufs_hw_queue *hwq = NULL;
2936
2937         switch (hba->ufshcd_state) {
2938         case UFSHCD_STATE_OPERATIONAL:
2939                 break;
2940         case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2941                 /*
2942                  * SCSI error handler can call ->queuecommand() while UFS error
2943                  * handler is in progress. Error interrupts could change the
2944                  * state from UFSHCD_STATE_RESET to
2945                  * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2946                  * being issued in that case.
2947                  */
2948                 if (ufshcd_eh_in_progress(hba)) {
2949                         err = SCSI_MLQUEUE_HOST_BUSY;
2950                         goto out;
2951                 }
2952                 break;
2953         case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2954                 /*
2955                  * pm_runtime_get_sync() is used at error handling preparation
2956                  * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2957                  * PM ops, it can never be finished if we let SCSI layer keep
2958                  * retrying it, which gets err handler stuck forever. Neither
2959                  * can we let the scsi cmd pass through, because UFS is in bad
2960                  * state, the scsi cmd may eventually time out, which will get
2961                  * err handler blocked for too long. So, just fail the scsi cmd
2962                  * sent from PM ops, err handler can recover PM error anyways.
2963                  */
2964                 if (hba->pm_op_in_progress) {
2965                         hba->force_reset = true;
2966                         set_host_byte(cmd, DID_BAD_TARGET);
2967                         scsi_done(cmd);
2968                         goto out;
2969                 }
2970                 fallthrough;
2971         case UFSHCD_STATE_RESET:
2972                 err = SCSI_MLQUEUE_HOST_BUSY;
2973                 goto out;
2974         case UFSHCD_STATE_ERROR:
2975                 set_host_byte(cmd, DID_ERROR);
2976                 scsi_done(cmd);
2977                 goto out;
2978         }
2979
2980         hba->req_abort_count = 0;
2981
2982         ufshcd_hold(hba);
2983
2984         lrbp = &hba->lrb[tag];
2985         lrbp->cmd = cmd;
2986         lrbp->task_tag = tag;
2987         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2988         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
2989
2990         ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
2991
2992         lrbp->req_abort_skip = false;
2993
2994         ufshcd_comp_scsi_upiu(hba, lrbp);
2995
2996         err = ufshcd_map_sg(hba, lrbp);
2997         if (err) {
2998                 ufshcd_release(hba);
2999                 goto out;
3000         }
3001
3002         if (is_mcq_enabled(hba))
3003                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
3004
3005         ufshcd_send_command(hba, tag, hwq);
3006
3007 out:
3008         if (ufs_trigger_eh(hba)) {
3009                 unsigned long flags;
3010
3011                 spin_lock_irqsave(hba->host->host_lock, flags);
3012                 ufshcd_schedule_eh_work(hba);
3013                 spin_unlock_irqrestore(hba->host->host_lock, flags);
3014         }
3015
3016         return err;
3017 }
3018
3019 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
3020                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
3021 {
3022         lrbp->cmd = NULL;
3023         lrbp->task_tag = tag;
3024         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
3025         lrbp->intr_cmd = true; /* No interrupt aggregation */
3026         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
3027         hba->dev_cmd.type = cmd_type;
3028
3029         return ufshcd_compose_devman_upiu(hba, lrbp);
3030 }
3031
3032 /*
3033  * Check with the block layer if the command is inflight
3034  * @cmd: command to check.
3035  *
3036  * Return: true if command is inflight; false if not.
3037  */
3038 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
3039 {
3040         struct request *rq;
3041
3042         if (!cmd)
3043                 return false;
3044
3045         rq = scsi_cmd_to_rq(cmd);
3046         if (!blk_mq_request_started(rq))
3047                 return false;
3048
3049         return true;
3050 }
3051
3052 /*
3053  * Clear the pending command in the controller and wait until
3054  * the controller confirms that the command has been cleared.
3055  * @hba: per adapter instance
3056  * @task_tag: The tag number of the command to be cleared.
3057  */
3058 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
3059 {
3060         u32 mask;
3061         unsigned long flags;
3062         int err;
3063
3064         if (is_mcq_enabled(hba)) {
3065                 /*
3066                  * MCQ mode. Clean up the MCQ resources similar to
3067                  * what the ufshcd_utrl_clear() does for SDB mode.
3068                  */
3069                 err = ufshcd_mcq_sq_cleanup(hba, task_tag);
3070                 if (err) {
3071                         dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
3072                                 __func__, task_tag, err);
3073                         return err;
3074                 }
3075                 return 0;
3076         }
3077
3078         mask = 1U << task_tag;
3079
3080         /* clear outstanding transaction before retry */
3081         spin_lock_irqsave(hba->host->host_lock, flags);
3082         ufshcd_utrl_clear(hba, mask);
3083         spin_unlock_irqrestore(hba->host->host_lock, flags);
3084
3085         /*
3086          * wait for h/w to clear corresponding bit in door-bell.
3087          * max. wait is 1 sec.
3088          */
3089         return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
3090                                         mask, ~mask, 1000, 1000);
3091 }
3092
3093 /**
3094  * ufshcd_dev_cmd_completion() - handles device management command responses
3095  * @hba: per adapter instance
3096  * @lrbp: pointer to local reference block
3097  *
3098  * Return: 0 upon success; < 0 upon failure.
3099  */
3100 static int
3101 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3102 {
3103         enum upiu_response_transaction resp;
3104         int err = 0;
3105
3106         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
3107         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3108
3109         switch (resp) {
3110         case UPIU_TRANSACTION_NOP_IN:
3111                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
3112                         err = -EINVAL;
3113                         dev_err(hba->dev, "%s: unexpected response %x\n",
3114                                         __func__, resp);
3115                 }
3116                 break;
3117         case UPIU_TRANSACTION_QUERY_RSP: {
3118                 u8 response = lrbp->ucd_rsp_ptr->header.response;
3119
3120                 if (response == 0)
3121                         err = ufshcd_copy_query_response(hba, lrbp);
3122                 break;
3123         }
3124         case UPIU_TRANSACTION_REJECT_UPIU:
3125                 /* TODO: handle Reject UPIU Response */
3126                 err = -EPERM;
3127                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
3128                                 __func__);
3129                 break;
3130         case UPIU_TRANSACTION_RESPONSE:
3131                 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) {
3132                         err = -EINVAL;
3133                         dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp);
3134                 }
3135                 break;
3136         default:
3137                 err = -EINVAL;
3138                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
3139                                 __func__, resp);
3140                 break;
3141         }
3142
3143         return err;
3144 }
3145
3146 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
3147                 struct ufshcd_lrb *lrbp, int max_timeout)
3148 {
3149         unsigned long time_left = msecs_to_jiffies(max_timeout);
3150         unsigned long flags;
3151         bool pending;
3152         int err;
3153
3154 retry:
3155         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
3156                                                 time_left);
3157
3158         if (likely(time_left)) {
3159                 /*
3160                  * The completion handler called complete() and the caller of
3161                  * this function still owns the @lrbp tag so the code below does
3162                  * not trigger any race conditions.
3163                  */
3164                 hba->dev_cmd.complete = NULL;
3165                 err = ufshcd_get_tr_ocs(lrbp, NULL);
3166                 if (!err)
3167                         err = ufshcd_dev_cmd_completion(hba, lrbp);
3168         } else {
3169                 err = -ETIMEDOUT;
3170                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3171                         __func__, lrbp->task_tag);
3172
3173                 /* MCQ mode */
3174                 if (is_mcq_enabled(hba)) {
3175                         err = ufshcd_clear_cmd(hba, lrbp->task_tag);
3176                         hba->dev_cmd.complete = NULL;
3177                         return err;
3178                 }
3179
3180                 /* SDB mode */
3181                 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
3182                         /* successfully cleared the command, retry if needed */
3183                         err = -EAGAIN;
3184                         /*
3185                          * Since clearing the command succeeded we also need to
3186                          * clear the task tag bit from the outstanding_reqs
3187                          * variable.
3188                          */
3189                         spin_lock_irqsave(&hba->outstanding_lock, flags);
3190                         pending = test_bit(lrbp->task_tag,
3191                                            &hba->outstanding_reqs);
3192                         if (pending) {
3193                                 hba->dev_cmd.complete = NULL;
3194                                 __clear_bit(lrbp->task_tag,
3195                                             &hba->outstanding_reqs);
3196                         }
3197                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3198
3199                         if (!pending) {
3200                                 /*
3201                                  * The completion handler ran while we tried to
3202                                  * clear the command.
3203                                  */
3204                                 time_left = 1;
3205                                 goto retry;
3206                         }
3207                 } else {
3208                         dev_err(hba->dev, "%s: failed to clear tag %d\n",
3209                                 __func__, lrbp->task_tag);
3210
3211                         spin_lock_irqsave(&hba->outstanding_lock, flags);
3212                         pending = test_bit(lrbp->task_tag,
3213                                            &hba->outstanding_reqs);
3214                         if (pending)
3215                                 hba->dev_cmd.complete = NULL;
3216                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3217
3218                         if (!pending) {
3219                                 /*
3220                                  * The completion handler ran while we tried to
3221                                  * clear the command.
3222                                  */
3223                                 time_left = 1;
3224                                 goto retry;
3225                         }
3226                 }
3227         }
3228
3229         return err;
3230 }
3231
3232 /**
3233  * ufshcd_exec_dev_cmd - API for sending device management requests
3234  * @hba: UFS hba
3235  * @cmd_type: specifies the type (NOP, Query...)
3236  * @timeout: timeout in milliseconds
3237  *
3238  * Return: 0 upon success; < 0 upon failure.
3239  *
3240  * NOTE: Since there is only one available tag for device management commands,
3241  * it is expected you hold the hba->dev_cmd.lock mutex.
3242  */
3243 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3244                 enum dev_cmd_type cmd_type, int timeout)
3245 {
3246         DECLARE_COMPLETION_ONSTACK(wait);
3247         const u32 tag = hba->reserved_slot;
3248         struct ufshcd_lrb *lrbp;
3249         int err;
3250
3251         /* Protects use of hba->reserved_slot. */
3252         lockdep_assert_held(&hba->dev_cmd.lock);
3253
3254         down_read(&hba->clk_scaling_lock);
3255
3256         lrbp = &hba->lrb[tag];
3257         lrbp->cmd = NULL;
3258         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3259         if (unlikely(err))
3260                 goto out;
3261
3262         hba->dev_cmd.complete = &wait;
3263
3264         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3265
3266         ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3267         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3268         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3269                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3270
3271 out:
3272         up_read(&hba->clk_scaling_lock);
3273         return err;
3274 }
3275
3276 /**
3277  * ufshcd_init_query() - init the query response and request parameters
3278  * @hba: per-adapter instance
3279  * @request: address of the request pointer to be initialized
3280  * @response: address of the response pointer to be initialized
3281  * @opcode: operation to perform
3282  * @idn: flag idn to access
3283  * @index: LU number to access
3284  * @selector: query/flag/descriptor further identification
3285  */
3286 static inline void ufshcd_init_query(struct ufs_hba *hba,
3287                 struct ufs_query_req **request, struct ufs_query_res **response,
3288                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3289 {
3290         *request = &hba->dev_cmd.query.request;
3291         *response = &hba->dev_cmd.query.response;
3292         memset(*request, 0, sizeof(struct ufs_query_req));
3293         memset(*response, 0, sizeof(struct ufs_query_res));
3294         (*request)->upiu_req.opcode = opcode;
3295         (*request)->upiu_req.idn = idn;
3296         (*request)->upiu_req.index = index;
3297         (*request)->upiu_req.selector = selector;
3298 }
3299
3300 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3301         enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3302 {
3303         int ret;
3304         int retries;
3305
3306         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3307                 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3308                 if (ret)
3309                         dev_dbg(hba->dev,
3310                                 "%s: failed with error %d, retries %d\n",
3311                                 __func__, ret, retries);
3312                 else
3313                         break;
3314         }
3315
3316         if (ret)
3317                 dev_err(hba->dev,
3318                         "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n",
3319                         __func__, opcode, idn, ret, retries);
3320         return ret;
3321 }
3322
3323 /**
3324  * ufshcd_query_flag() - API function for sending flag query requests
3325  * @hba: per-adapter instance
3326  * @opcode: flag query to perform
3327  * @idn: flag idn to access
3328  * @index: flag index to access
3329  * @flag_res: the flag value after the query request completes
3330  *
3331  * Return: 0 for success, non-zero in case of failure.
3332  */
3333 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3334                         enum flag_idn idn, u8 index, bool *flag_res)
3335 {
3336         struct ufs_query_req *request = NULL;
3337         struct ufs_query_res *response = NULL;
3338         int err, selector = 0;
3339         int timeout = QUERY_REQ_TIMEOUT;
3340
3341         BUG_ON(!hba);
3342
3343         ufshcd_hold(hba);
3344         mutex_lock(&hba->dev_cmd.lock);
3345         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3346                         selector);
3347
3348         switch (opcode) {
3349         case UPIU_QUERY_OPCODE_SET_FLAG:
3350         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3351         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3352                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3353                 break;
3354         case UPIU_QUERY_OPCODE_READ_FLAG:
3355                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3356                 if (!flag_res) {
3357                         /* No dummy reads */
3358                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
3359                                         __func__);
3360                         err = -EINVAL;
3361                         goto out_unlock;
3362                 }
3363                 break;
3364         default:
3365                 dev_err(hba->dev,
3366                         "%s: Expected query flag opcode but got = %d\n",
3367                         __func__, opcode);
3368                 err = -EINVAL;
3369                 goto out_unlock;
3370         }
3371
3372         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3373
3374         if (err) {
3375                 dev_err(hba->dev,
3376                         "%s: Sending flag query for idn %d failed, err = %d\n",
3377                         __func__, idn, err);
3378                 goto out_unlock;
3379         }
3380
3381         if (flag_res)
3382                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3383                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3384
3385 out_unlock:
3386         mutex_unlock(&hba->dev_cmd.lock);
3387         ufshcd_release(hba);
3388         return err;
3389 }
3390
3391 /**
3392  * ufshcd_query_attr - API function for sending attribute requests
3393  * @hba: per-adapter instance
3394  * @opcode: attribute opcode
3395  * @idn: attribute idn to access
3396  * @index: index field
3397  * @selector: selector field
3398  * @attr_val: the attribute value after the query request completes
3399  *
3400  * Return: 0 for success, non-zero in case of failure.
3401 */
3402 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3403                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3404 {
3405         struct ufs_query_req *request = NULL;
3406         struct ufs_query_res *response = NULL;
3407         int err;
3408
3409         BUG_ON(!hba);
3410
3411         if (!attr_val) {
3412                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3413                                 __func__, opcode);
3414                 return -EINVAL;
3415         }
3416
3417         ufshcd_hold(hba);
3418
3419         mutex_lock(&hba->dev_cmd.lock);
3420         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3421                         selector);
3422
3423         switch (opcode) {
3424         case UPIU_QUERY_OPCODE_WRITE_ATTR:
3425                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3426                 request->upiu_req.value = cpu_to_be32(*attr_val);
3427                 break;
3428         case UPIU_QUERY_OPCODE_READ_ATTR:
3429                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3430                 break;
3431         default:
3432                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3433                                 __func__, opcode);
3434                 err = -EINVAL;
3435                 goto out_unlock;
3436         }
3437
3438         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3439
3440         if (err) {
3441                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3442                                 __func__, opcode, idn, index, err);
3443                 goto out_unlock;
3444         }
3445
3446         *attr_val = be32_to_cpu(response->upiu_res.value);
3447
3448 out_unlock:
3449         mutex_unlock(&hba->dev_cmd.lock);
3450         ufshcd_release(hba);
3451         return err;
3452 }
3453
3454 /**
3455  * ufshcd_query_attr_retry() - API function for sending query
3456  * attribute with retries
3457  * @hba: per-adapter instance
3458  * @opcode: attribute opcode
3459  * @idn: attribute idn to access
3460  * @index: index field
3461  * @selector: selector field
3462  * @attr_val: the attribute value after the query request
3463  * completes
3464  *
3465  * Return: 0 for success, non-zero in case of failure.
3466 */
3467 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3468         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3469         u32 *attr_val)
3470 {
3471         int ret = 0;
3472         u32 retries;
3473
3474         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3475                 ret = ufshcd_query_attr(hba, opcode, idn, index,
3476                                                 selector, attr_val);
3477                 if (ret)
3478                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3479                                 __func__, ret, retries);
3480                 else
3481                         break;
3482         }
3483
3484         if (ret)
3485                 dev_err(hba->dev,
3486                         "%s: query attribute, idn %d, failed with error %d after %d retries\n",
3487                         __func__, idn, ret, QUERY_REQ_RETRIES);
3488         return ret;
3489 }
3490
3491 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3492                         enum query_opcode opcode, enum desc_idn idn, u8 index,
3493                         u8 selector, u8 *desc_buf, int *buf_len)
3494 {
3495         struct ufs_query_req *request = NULL;
3496         struct ufs_query_res *response = NULL;
3497         int err;
3498
3499         BUG_ON(!hba);
3500
3501         if (!desc_buf) {
3502                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3503                                 __func__, opcode);
3504                 return -EINVAL;
3505         }
3506
3507         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3508                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3509                                 __func__, *buf_len);
3510                 return -EINVAL;
3511         }
3512
3513         ufshcd_hold(hba);
3514
3515         mutex_lock(&hba->dev_cmd.lock);
3516         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3517                         selector);
3518         hba->dev_cmd.query.descriptor = desc_buf;
3519         request->upiu_req.length = cpu_to_be16(*buf_len);
3520
3521         switch (opcode) {
3522         case UPIU_QUERY_OPCODE_WRITE_DESC:
3523                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3524                 break;
3525         case UPIU_QUERY_OPCODE_READ_DESC:
3526                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3527                 break;
3528         default:
3529                 dev_err(hba->dev,
3530                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3531                                 __func__, opcode);
3532                 err = -EINVAL;
3533                 goto out_unlock;
3534         }
3535
3536         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3537
3538         if (err) {
3539                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3540                                 __func__, opcode, idn, index, err);
3541                 goto out_unlock;
3542         }
3543
3544         *buf_len = be16_to_cpu(response->upiu_res.length);
3545
3546 out_unlock:
3547         hba->dev_cmd.query.descriptor = NULL;
3548         mutex_unlock(&hba->dev_cmd.lock);
3549         ufshcd_release(hba);
3550         return err;
3551 }
3552
3553 /**
3554  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3555  * @hba: per-adapter instance
3556  * @opcode: attribute opcode
3557  * @idn: attribute idn to access
3558  * @index: index field
3559  * @selector: selector field
3560  * @desc_buf: the buffer that contains the descriptor
3561  * @buf_len: length parameter passed to the device
3562  *
3563  * The buf_len parameter will contain, on return, the length parameter
3564  * received on the response.
3565  *
3566  * Return: 0 for success, non-zero in case of failure.
3567  */
3568 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3569                                   enum query_opcode opcode,
3570                                   enum desc_idn idn, u8 index,
3571                                   u8 selector,
3572                                   u8 *desc_buf, int *buf_len)
3573 {
3574         int err;
3575         int retries;
3576
3577         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3578                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3579                                                 selector, desc_buf, buf_len);
3580                 if (!err || err == -EINVAL)
3581                         break;
3582         }
3583
3584         return err;
3585 }
3586
3587 /**
3588  * ufshcd_read_desc_param - read the specified descriptor parameter
3589  * @hba: Pointer to adapter instance
3590  * @desc_id: descriptor idn value
3591  * @desc_index: descriptor index
3592  * @param_offset: offset of the parameter to read
3593  * @param_read_buf: pointer to buffer where parameter would be read
3594  * @param_size: sizeof(param_read_buf)
3595  *
3596  * Return: 0 in case of success, non-zero otherwise.
3597  */
3598 int ufshcd_read_desc_param(struct ufs_hba *hba,
3599                            enum desc_idn desc_id,
3600                            int desc_index,
3601                            u8 param_offset,
3602                            u8 *param_read_buf,
3603                            u8 param_size)
3604 {
3605         int ret;
3606         u8 *desc_buf;
3607         int buff_len = QUERY_DESC_MAX_SIZE;
3608         bool is_kmalloc = true;
3609
3610         /* Safety check */
3611         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3612                 return -EINVAL;
3613
3614         /* Check whether we need temp memory */
3615         if (param_offset != 0 || param_size < buff_len) {
3616                 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3617                 if (!desc_buf)
3618                         return -ENOMEM;
3619         } else {
3620                 desc_buf = param_read_buf;
3621                 is_kmalloc = false;
3622         }
3623
3624         /* Request for full descriptor */
3625         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3626                                             desc_id, desc_index, 0,
3627                                             desc_buf, &buff_len);
3628         if (ret) {
3629                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3630                         __func__, desc_id, desc_index, param_offset, ret);
3631                 goto out;
3632         }
3633
3634         /* Update descriptor length */
3635         buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3636
3637         if (param_offset >= buff_len) {
3638                 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3639                         __func__, param_offset, desc_id, buff_len);
3640                 ret = -EINVAL;
3641                 goto out;
3642         }
3643
3644         /* Sanity check */
3645         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3646                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3647                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3648                 ret = -EINVAL;
3649                 goto out;
3650         }
3651
3652         if (is_kmalloc) {
3653                 /* Make sure we don't copy more data than available */
3654                 if (param_offset >= buff_len)
3655                         ret = -EINVAL;
3656                 else
3657                         memcpy(param_read_buf, &desc_buf[param_offset],
3658                                min_t(u32, param_size, buff_len - param_offset));
3659         }
3660 out:
3661         if (is_kmalloc)
3662                 kfree(desc_buf);
3663         return ret;
3664 }
3665
3666 /**
3667  * struct uc_string_id - unicode string
3668  *
3669  * @len: size of this descriptor inclusive
3670  * @type: descriptor type
3671  * @uc: unicode string character
3672  */
3673 struct uc_string_id {
3674         u8 len;
3675         u8 type;
3676         wchar_t uc[];
3677 } __packed;
3678
3679 /* replace non-printable or non-ASCII characters with spaces */
3680 static inline char ufshcd_remove_non_printable(u8 ch)
3681 {
3682         return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3683 }
3684
3685 /**
3686  * ufshcd_read_string_desc - read string descriptor
3687  * @hba: pointer to adapter instance
3688  * @desc_index: descriptor index
3689  * @buf: pointer to buffer where descriptor would be read,
3690  *       the caller should free the memory.
3691  * @ascii: if true convert from unicode to ascii characters
3692  *         null terminated string.
3693  *
3694  * Return:
3695  * *      string size on success.
3696  * *      -ENOMEM: on allocation failure
3697  * *      -EINVAL: on a wrong parameter
3698  */
3699 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3700                             u8 **buf, bool ascii)
3701 {
3702         struct uc_string_id *uc_str;
3703         u8 *str;
3704         int ret;
3705
3706         if (!buf)
3707                 return -EINVAL;
3708
3709         uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3710         if (!uc_str)
3711                 return -ENOMEM;
3712
3713         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3714                                      (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3715         if (ret < 0) {
3716                 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3717                         QUERY_REQ_RETRIES, ret);
3718                 str = NULL;
3719                 goto out;
3720         }
3721
3722         if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3723                 dev_dbg(hba->dev, "String Desc is of zero length\n");
3724                 str = NULL;
3725                 ret = 0;
3726                 goto out;
3727         }
3728
3729         if (ascii) {
3730                 ssize_t ascii_len;
3731                 int i;
3732                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3733                 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3734                 str = kzalloc(ascii_len, GFP_KERNEL);
3735                 if (!str) {
3736                         ret = -ENOMEM;
3737                         goto out;
3738                 }
3739
3740                 /*
3741                  * the descriptor contains string in UTF16 format
3742                  * we need to convert to utf-8 so it can be displayed
3743                  */
3744                 ret = utf16s_to_utf8s(uc_str->uc,
3745                                       uc_str->len - QUERY_DESC_HDR_SIZE,
3746                                       UTF16_BIG_ENDIAN, str, ascii_len - 1);
3747
3748                 /* replace non-printable or non-ASCII characters with spaces */
3749                 for (i = 0; i < ret; i++)
3750                         str[i] = ufshcd_remove_non_printable(str[i]);
3751
3752                 str[ret++] = '\0';
3753
3754         } else {
3755                 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3756                 if (!str) {
3757                         ret = -ENOMEM;
3758                         goto out;
3759                 }
3760                 ret = uc_str->len;
3761         }
3762 out:
3763         *buf = str;
3764         kfree(uc_str);
3765         return ret;
3766 }
3767
3768 /**
3769  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3770  * @hba: Pointer to adapter instance
3771  * @lun: lun id
3772  * @param_offset: offset of the parameter to read
3773  * @param_read_buf: pointer to buffer where parameter would be read
3774  * @param_size: sizeof(param_read_buf)
3775  *
3776  * Return: 0 in case of success, non-zero otherwise.
3777  */
3778 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3779                                               int lun,
3780                                               enum unit_desc_param param_offset,
3781                                               u8 *param_read_buf,
3782                                               u32 param_size)
3783 {
3784         /*
3785          * Unit descriptors are only available for general purpose LUs (LUN id
3786          * from 0 to 7) and RPMB Well known LU.
3787          */
3788         if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3789                 return -EOPNOTSUPP;
3790
3791         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3792                                       param_offset, param_read_buf, param_size);
3793 }
3794
3795 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3796 {
3797         int err = 0;
3798         u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3799
3800         if (hba->dev_info.wspecversion >= 0x300) {
3801                 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3802                                 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3803                                 &gating_wait);
3804                 if (err)
3805                         dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3806                                          err, gating_wait);
3807
3808                 if (gating_wait == 0) {
3809                         gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3810                         dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3811                                          gating_wait);
3812                 }
3813
3814                 hba->dev_info.clk_gating_wait_us = gating_wait;
3815         }
3816
3817         return err;
3818 }
3819
3820 /**
3821  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3822  * @hba: per adapter instance
3823  *
3824  * 1. Allocate DMA memory for Command Descriptor array
3825  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3826  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3827  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3828  *      (UTMRDL)
3829  * 4. Allocate memory for local reference block(lrb).
3830  *
3831  * Return: 0 for success, non-zero in case of failure.
3832  */
3833 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3834 {
3835         size_t utmrdl_size, utrdl_size, ucdl_size;
3836
3837         /* Allocate memory for UTP command descriptors */
3838         ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs;
3839         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3840                                                   ucdl_size,
3841                                                   &hba->ucdl_dma_addr,
3842                                                   GFP_KERNEL);
3843
3844         /*
3845          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3846          */
3847         if (!hba->ucdl_base_addr ||
3848             WARN_ON(hba->ucdl_dma_addr & (128 - 1))) {
3849                 dev_err(hba->dev,
3850                         "Command Descriptor Memory allocation failed\n");
3851                 goto out;
3852         }
3853
3854         /*
3855          * Allocate memory for UTP Transfer descriptors
3856          * UFSHCI requires 1KB alignment of UTRD
3857          */
3858         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3859         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3860                                                    utrdl_size,
3861                                                    &hba->utrdl_dma_addr,
3862                                                    GFP_KERNEL);
3863         if (!hba->utrdl_base_addr ||
3864             WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
3865                 dev_err(hba->dev,
3866                         "Transfer Descriptor Memory allocation failed\n");
3867                 goto out;
3868         }
3869
3870         /*
3871          * Skip utmrdl allocation; it may have been
3872          * allocated during first pass and not released during
3873          * MCQ memory allocation.
3874          * See ufshcd_release_sdb_queue() and ufshcd_config_mcq()
3875          */
3876         if (hba->utmrdl_base_addr)
3877                 goto skip_utmrdl;
3878         /*
3879          * Allocate memory for UTP Task Management descriptors
3880          * UFSHCI requires 1KB alignment of UTMRD
3881          */
3882         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3883         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3884                                                     utmrdl_size,
3885                                                     &hba->utmrdl_dma_addr,
3886                                                     GFP_KERNEL);
3887         if (!hba->utmrdl_base_addr ||
3888             WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
3889                 dev_err(hba->dev,
3890                 "Task Management Descriptor Memory allocation failed\n");
3891                 goto out;
3892         }
3893
3894 skip_utmrdl:
3895         /* Allocate memory for local reference block */
3896         hba->lrb = devm_kcalloc(hba->dev,
3897                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3898                                 GFP_KERNEL);
3899         if (!hba->lrb) {
3900                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3901                 goto out;
3902         }
3903         return 0;
3904 out:
3905         return -ENOMEM;
3906 }
3907
3908 /**
3909  * ufshcd_host_memory_configure - configure local reference block with
3910  *                              memory offsets
3911  * @hba: per adapter instance
3912  *
3913  * Configure Host memory space
3914  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3915  * address.
3916  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3917  * and PRDT offset.
3918  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3919  * into local reference block.
3920  */
3921 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3922 {
3923         struct utp_transfer_req_desc *utrdlp;
3924         dma_addr_t cmd_desc_dma_addr;
3925         dma_addr_t cmd_desc_element_addr;
3926         u16 response_offset;
3927         u16 prdt_offset;
3928         int cmd_desc_size;
3929         int i;
3930
3931         utrdlp = hba->utrdl_base_addr;
3932
3933         response_offset =
3934                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3935         prdt_offset =
3936                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3937
3938         cmd_desc_size = ufshcd_get_ucd_size(hba);
3939         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3940
3941         for (i = 0; i < hba->nutrs; i++) {
3942                 /* Configure UTRD with command descriptor base address */
3943                 cmd_desc_element_addr =
3944                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3945                 utrdlp[i].command_desc_base_addr =
3946                                 cpu_to_le64(cmd_desc_element_addr);
3947
3948                 /* Response upiu and prdt offset should be in double words */
3949                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3950                         utrdlp[i].response_upiu_offset =
3951                                 cpu_to_le16(response_offset);
3952                         utrdlp[i].prd_table_offset =
3953                                 cpu_to_le16(prdt_offset);
3954                         utrdlp[i].response_upiu_length =
3955                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3956                 } else {
3957                         utrdlp[i].response_upiu_offset =
3958                                 cpu_to_le16(response_offset >> 2);
3959                         utrdlp[i].prd_table_offset =
3960                                 cpu_to_le16(prdt_offset >> 2);
3961                         utrdlp[i].response_upiu_length =
3962                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3963                 }
3964
3965                 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3966         }
3967 }
3968
3969 /**
3970  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3971  * @hba: per adapter instance
3972  *
3973  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3974  * in order to initialize the Unipro link startup procedure.
3975  * Once the Unipro links are up, the device connected to the controller
3976  * is detected.
3977  *
3978  * Return: 0 on success, non-zero value on failure.
3979  */
3980 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3981 {
3982         struct uic_command uic_cmd = {0};
3983         int ret;
3984
3985         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3986
3987         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3988         if (ret)
3989                 dev_dbg(hba->dev,
3990                         "dme-link-startup: error code %d\n", ret);
3991         return ret;
3992 }
3993 /**
3994  * ufshcd_dme_reset - UIC command for DME_RESET
3995  * @hba: per adapter instance
3996  *
3997  * DME_RESET command is issued in order to reset UniPro stack.
3998  * This function now deals with cold reset.
3999  *
4000  * Return: 0 on success, non-zero value on failure.
4001  */
4002 static int ufshcd_dme_reset(struct ufs_hba *hba)
4003 {
4004         struct uic_command uic_cmd = {0};
4005         int ret;
4006
4007         uic_cmd.command = UIC_CMD_DME_RESET;
4008
4009         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4010         if (ret)
4011                 dev_err(hba->dev,
4012                         "dme-reset: error code %d\n", ret);
4013
4014         return ret;
4015 }
4016
4017 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
4018                                int agreed_gear,
4019                                int adapt_val)
4020 {
4021         int ret;
4022
4023         if (agreed_gear < UFS_HS_G4)
4024                 adapt_val = PA_NO_ADAPT;
4025
4026         ret = ufshcd_dme_set(hba,
4027                              UIC_ARG_MIB(PA_TXHSADAPTTYPE),
4028                              adapt_val);
4029         return ret;
4030 }
4031 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
4032
4033 /**
4034  * ufshcd_dme_enable - UIC command for DME_ENABLE
4035  * @hba: per adapter instance
4036  *
4037  * DME_ENABLE command is issued in order to enable UniPro stack.
4038  *
4039  * Return: 0 on success, non-zero value on failure.
4040  */
4041 static int ufshcd_dme_enable(struct ufs_hba *hba)
4042 {
4043         struct uic_command uic_cmd = {0};
4044         int ret;
4045
4046         uic_cmd.command = UIC_CMD_DME_ENABLE;
4047
4048         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4049         if (ret)
4050                 dev_err(hba->dev,
4051                         "dme-enable: error code %d\n", ret);
4052
4053         return ret;
4054 }
4055
4056 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
4057 {
4058         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
4059         unsigned long min_sleep_time_us;
4060
4061         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
4062                 return;
4063
4064         /*
4065          * last_dme_cmd_tstamp will be 0 only for 1st call to
4066          * this function
4067          */
4068         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
4069                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
4070         } else {
4071                 unsigned long delta =
4072                         (unsigned long) ktime_to_us(
4073                                 ktime_sub(ktime_get(),
4074                                 hba->last_dme_cmd_tstamp));
4075
4076                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
4077                         min_sleep_time_us =
4078                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
4079                 else
4080                         return; /* no more delay required */
4081         }
4082
4083         /* allow sleep for extra 50us if needed */
4084         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
4085 }
4086
4087 /**
4088  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
4089  * @hba: per adapter instance
4090  * @attr_sel: uic command argument1
4091  * @attr_set: attribute set type as uic command argument2
4092  * @mib_val: setting value as uic command argument3
4093  * @peer: indicate whether peer or local
4094  *
4095  * Return: 0 on success, non-zero value on failure.
4096  */
4097 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
4098                         u8 attr_set, u32 mib_val, u8 peer)
4099 {
4100         struct uic_command uic_cmd = {0};
4101         static const char *const action[] = {
4102                 "dme-set",
4103                 "dme-peer-set"
4104         };
4105         const char *set = action[!!peer];
4106         int ret;
4107         int retries = UFS_UIC_COMMAND_RETRIES;
4108
4109         uic_cmd.command = peer ?
4110                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
4111         uic_cmd.argument1 = attr_sel;
4112         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
4113         uic_cmd.argument3 = mib_val;
4114
4115         do {
4116                 /* for peer attributes we retry upon failure */
4117                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4118                 if (ret)
4119                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
4120                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
4121         } while (ret && peer && --retries);
4122
4123         if (ret)
4124                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
4125                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
4126                         UFS_UIC_COMMAND_RETRIES - retries);
4127
4128         return ret;
4129 }
4130 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
4131
4132 /**
4133  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
4134  * @hba: per adapter instance
4135  * @attr_sel: uic command argument1
4136  * @mib_val: the value of the attribute as returned by the UIC command
4137  * @peer: indicate whether peer or local
4138  *
4139  * Return: 0 on success, non-zero value on failure.
4140  */
4141 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
4142                         u32 *mib_val, u8 peer)
4143 {
4144         struct uic_command uic_cmd = {0};
4145         static const char *const action[] = {
4146                 "dme-get",
4147                 "dme-peer-get"
4148         };
4149         const char *get = action[!!peer];
4150         int ret;
4151         int retries = UFS_UIC_COMMAND_RETRIES;
4152         struct ufs_pa_layer_attr orig_pwr_info;
4153         struct ufs_pa_layer_attr temp_pwr_info;
4154         bool pwr_mode_change = false;
4155
4156         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
4157                 orig_pwr_info = hba->pwr_info;
4158                 temp_pwr_info = orig_pwr_info;
4159
4160                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
4161                     orig_pwr_info.pwr_rx == FAST_MODE) {
4162                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
4163                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
4164                         pwr_mode_change = true;
4165                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
4166                     orig_pwr_info.pwr_rx == SLOW_MODE) {
4167                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
4168                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
4169                         pwr_mode_change = true;
4170                 }
4171                 if (pwr_mode_change) {
4172                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
4173                         if (ret)
4174                                 goto out;
4175                 }
4176         }
4177
4178         uic_cmd.command = peer ?
4179                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
4180         uic_cmd.argument1 = attr_sel;
4181
4182         do {
4183                 /* for peer attributes we retry upon failure */
4184                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4185                 if (ret)
4186                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4187                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
4188         } while (ret && peer && --retries);
4189
4190         if (ret)
4191                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4192                         get, UIC_GET_ATTR_ID(attr_sel),
4193                         UFS_UIC_COMMAND_RETRIES - retries);
4194
4195         if (mib_val && !ret)
4196                 *mib_val = uic_cmd.argument3;
4197
4198         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4199             && pwr_mode_change)
4200                 ufshcd_change_power_mode(hba, &orig_pwr_info);
4201 out:
4202         return ret;
4203 }
4204 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4205
4206 /**
4207  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4208  * state) and waits for it to take effect.
4209  *
4210  * @hba: per adapter instance
4211  * @cmd: UIC command to execute
4212  *
4213  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4214  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4215  * and device UniPro link and hence it's final completion would be indicated by
4216  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4217  * addition to normal UIC command completion Status (UCCS). This function only
4218  * returns after the relevant status bits indicate the completion.
4219  *
4220  * Return: 0 on success, non-zero value on failure.
4221  */
4222 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4223 {
4224         DECLARE_COMPLETION_ONSTACK(uic_async_done);
4225         unsigned long flags;
4226         u8 status;
4227         int ret;
4228         bool reenable_intr = false;
4229
4230         mutex_lock(&hba->uic_cmd_mutex);
4231         ufshcd_add_delay_before_dme_cmd(hba);
4232
4233         spin_lock_irqsave(hba->host->host_lock, flags);
4234         if (ufshcd_is_link_broken(hba)) {
4235                 ret = -ENOLINK;
4236                 goto out_unlock;
4237         }
4238         hba->uic_async_done = &uic_async_done;
4239         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4240                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4241                 /*
4242                  * Make sure UIC command completion interrupt is disabled before
4243                  * issuing UIC command.
4244                  */
4245                 wmb();
4246                 reenable_intr = true;
4247         }
4248         spin_unlock_irqrestore(hba->host->host_lock, flags);
4249         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4250         if (ret) {
4251                 dev_err(hba->dev,
4252                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4253                         cmd->command, cmd->argument3, ret);
4254                 goto out;
4255         }
4256
4257         if (!wait_for_completion_timeout(hba->uic_async_done,
4258                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
4259                 dev_err(hba->dev,
4260                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4261                         cmd->command, cmd->argument3);
4262
4263                 if (!cmd->cmd_active) {
4264                         dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4265                                 __func__);
4266                         goto check_upmcrs;
4267                 }
4268
4269                 ret = -ETIMEDOUT;
4270                 goto out;
4271         }
4272
4273 check_upmcrs:
4274         status = ufshcd_get_upmcrs(hba);
4275         if (status != PWR_LOCAL) {
4276                 dev_err(hba->dev,
4277                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4278                         cmd->command, status);
4279                 ret = (status != PWR_OK) ? status : -1;
4280         }
4281 out:
4282         if (ret) {
4283                 ufshcd_print_host_state(hba);
4284                 ufshcd_print_pwr_info(hba);
4285                 ufshcd_print_evt_hist(hba);
4286         }
4287
4288         spin_lock_irqsave(hba->host->host_lock, flags);
4289         hba->active_uic_cmd = NULL;
4290         hba->uic_async_done = NULL;
4291         if (reenable_intr)
4292                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4293         if (ret) {
4294                 ufshcd_set_link_broken(hba);
4295                 ufshcd_schedule_eh_work(hba);
4296         }
4297 out_unlock:
4298         spin_unlock_irqrestore(hba->host->host_lock, flags);
4299         mutex_unlock(&hba->uic_cmd_mutex);
4300
4301         return ret;
4302 }
4303
4304 /**
4305  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4306  *                              using DME_SET primitives.
4307  * @hba: per adapter instance
4308  * @mode: powr mode value
4309  *
4310  * Return: 0 on success, non-zero value on failure.
4311  */
4312 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4313 {
4314         struct uic_command uic_cmd = {0};
4315         int ret;
4316
4317         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4318                 ret = ufshcd_dme_set(hba,
4319                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4320                 if (ret) {
4321                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4322                                                 __func__, ret);
4323                         goto out;
4324                 }
4325         }
4326
4327         uic_cmd.command = UIC_CMD_DME_SET;
4328         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4329         uic_cmd.argument3 = mode;
4330         ufshcd_hold(hba);
4331         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4332         ufshcd_release(hba);
4333
4334 out:
4335         return ret;
4336 }
4337 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode);
4338
4339 int ufshcd_link_recovery(struct ufs_hba *hba)
4340 {
4341         int ret;
4342         unsigned long flags;
4343
4344         spin_lock_irqsave(hba->host->host_lock, flags);
4345         hba->ufshcd_state = UFSHCD_STATE_RESET;
4346         ufshcd_set_eh_in_progress(hba);
4347         spin_unlock_irqrestore(hba->host->host_lock, flags);
4348
4349         /* Reset the attached device */
4350         ufshcd_device_reset(hba);
4351
4352         ret = ufshcd_host_reset_and_restore(hba);
4353
4354         spin_lock_irqsave(hba->host->host_lock, flags);
4355         if (ret)
4356                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4357         ufshcd_clear_eh_in_progress(hba);
4358         spin_unlock_irqrestore(hba->host->host_lock, flags);
4359
4360         if (ret)
4361                 dev_err(hba->dev, "%s: link recovery failed, err %d",
4362                         __func__, ret);
4363
4364         return ret;
4365 }
4366 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4367
4368 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4369 {
4370         int ret;
4371         struct uic_command uic_cmd = {0};
4372         ktime_t start = ktime_get();
4373
4374         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4375
4376         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4377         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4378         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4379                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4380
4381         if (ret)
4382                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4383                         __func__, ret);
4384         else
4385                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4386                                                                 POST_CHANGE);
4387
4388         return ret;
4389 }
4390 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4391
4392 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4393 {
4394         struct uic_command uic_cmd = {0};
4395         int ret;
4396         ktime_t start = ktime_get();
4397
4398         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4399
4400         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4401         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4402         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4403                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4404
4405         if (ret) {
4406                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4407                         __func__, ret);
4408         } else {
4409                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4410                                                                 POST_CHANGE);
4411                 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4412                 hba->ufs_stats.hibern8_exit_cnt++;
4413         }
4414
4415         return ret;
4416 }
4417 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4418
4419 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba)
4420 {
4421         if (!ufshcd_is_auto_hibern8_supported(hba))
4422                 return;
4423
4424         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4425 }
4426
4427 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4428 {
4429         const u32 cur_ahit = READ_ONCE(hba->ahit);
4430
4431         if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit)
4432                 return;
4433
4434         WRITE_ONCE(hba->ahit, ahit);
4435         if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4436                 ufshcd_rpm_get_sync(hba);
4437                 ufshcd_hold(hba);
4438                 ufshcd_configure_auto_hibern8(hba);
4439                 ufshcd_release(hba);
4440                 ufshcd_rpm_put_sync(hba);
4441         }
4442 }
4443 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4444
4445  /**
4446  * ufshcd_init_pwr_info - setting the POR (power on reset)
4447  * values in hba power info
4448  * @hba: per-adapter instance
4449  */
4450 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4451 {
4452         hba->pwr_info.gear_rx = UFS_PWM_G1;
4453         hba->pwr_info.gear_tx = UFS_PWM_G1;
4454         hba->pwr_info.lane_rx = UFS_LANE_1;
4455         hba->pwr_info.lane_tx = UFS_LANE_1;
4456         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4457         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4458         hba->pwr_info.hs_rate = 0;
4459 }
4460
4461 /**
4462  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4463  * @hba: per-adapter instance
4464  *
4465  * Return: 0 upon success; < 0 upon failure.
4466  */
4467 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4468 {
4469         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4470
4471         if (hba->max_pwr_info.is_valid)
4472                 return 0;
4473
4474         if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4475                 pwr_info->pwr_tx = FASTAUTO_MODE;
4476                 pwr_info->pwr_rx = FASTAUTO_MODE;
4477         } else {
4478                 pwr_info->pwr_tx = FAST_MODE;
4479                 pwr_info->pwr_rx = FAST_MODE;
4480         }
4481         pwr_info->hs_rate = PA_HS_MODE_B;
4482
4483         /* Get the connected lane count */
4484         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4485                         &pwr_info->lane_rx);
4486         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4487                         &pwr_info->lane_tx);
4488
4489         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4490                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4491                                 __func__,
4492                                 pwr_info->lane_rx,
4493                                 pwr_info->lane_tx);
4494                 return -EINVAL;
4495         }
4496
4497         /*
4498          * First, get the maximum gears of HS speed.
4499          * If a zero value, it means there is no HSGEAR capability.
4500          * Then, get the maximum gears of PWM speed.
4501          */
4502         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4503         if (!pwr_info->gear_rx) {
4504                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4505                                 &pwr_info->gear_rx);
4506                 if (!pwr_info->gear_rx) {
4507                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4508                                 __func__, pwr_info->gear_rx);
4509                         return -EINVAL;
4510                 }
4511                 pwr_info->pwr_rx = SLOW_MODE;
4512         }
4513
4514         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4515                         &pwr_info->gear_tx);
4516         if (!pwr_info->gear_tx) {
4517                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4518                                 &pwr_info->gear_tx);
4519                 if (!pwr_info->gear_tx) {
4520                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4521                                 __func__, pwr_info->gear_tx);
4522                         return -EINVAL;
4523                 }
4524                 pwr_info->pwr_tx = SLOW_MODE;
4525         }
4526
4527         hba->max_pwr_info.is_valid = true;
4528         return 0;
4529 }
4530
4531 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4532                              struct ufs_pa_layer_attr *pwr_mode)
4533 {
4534         int ret;
4535
4536         /* if already configured to the requested pwr_mode */
4537         if (!hba->force_pmc &&
4538             pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4539             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4540             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4541             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4542             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4543             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4544             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4545                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4546                 return 0;
4547         }
4548
4549         /*
4550          * Configure attributes for power mode change with below.
4551          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4552          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4553          * - PA_HSSERIES
4554          */
4555         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4556         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4557                         pwr_mode->lane_rx);
4558         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4559                         pwr_mode->pwr_rx == FAST_MODE)
4560                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4561         else
4562                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4563
4564         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4565         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4566                         pwr_mode->lane_tx);
4567         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4568                         pwr_mode->pwr_tx == FAST_MODE)
4569                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4570         else
4571                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4572
4573         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4574             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4575             pwr_mode->pwr_rx == FAST_MODE ||
4576             pwr_mode->pwr_tx == FAST_MODE)
4577                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4578                                                 pwr_mode->hs_rate);
4579
4580         if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4581                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4582                                 DL_FC0ProtectionTimeOutVal_Default);
4583                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4584                                 DL_TC0ReplayTimeOutVal_Default);
4585                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4586                                 DL_AFC0ReqTimeOutVal_Default);
4587                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4588                                 DL_FC1ProtectionTimeOutVal_Default);
4589                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4590                                 DL_TC1ReplayTimeOutVal_Default);
4591                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4592                                 DL_AFC1ReqTimeOutVal_Default);
4593
4594                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4595                                 DL_FC0ProtectionTimeOutVal_Default);
4596                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4597                                 DL_TC0ReplayTimeOutVal_Default);
4598                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4599                                 DL_AFC0ReqTimeOutVal_Default);
4600         }
4601
4602         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4603                         | pwr_mode->pwr_tx);
4604
4605         if (ret) {
4606                 dev_err(hba->dev,
4607                         "%s: power mode change failed %d\n", __func__, ret);
4608         } else {
4609                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4610                                                                 pwr_mode);
4611
4612                 memcpy(&hba->pwr_info, pwr_mode,
4613                         sizeof(struct ufs_pa_layer_attr));
4614         }
4615
4616         return ret;
4617 }
4618
4619 /**
4620  * ufshcd_config_pwr_mode - configure a new power mode
4621  * @hba: per-adapter instance
4622  * @desired_pwr_mode: desired power configuration
4623  *
4624  * Return: 0 upon success; < 0 upon failure.
4625  */
4626 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4627                 struct ufs_pa_layer_attr *desired_pwr_mode)
4628 {
4629         struct ufs_pa_layer_attr final_params = { 0 };
4630         int ret;
4631
4632         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4633                                         desired_pwr_mode, &final_params);
4634
4635         if (ret)
4636                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4637
4638         ret = ufshcd_change_power_mode(hba, &final_params);
4639
4640         return ret;
4641 }
4642 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4643
4644 /**
4645  * ufshcd_complete_dev_init() - checks device readiness
4646  * @hba: per-adapter instance
4647  *
4648  * Set fDeviceInit flag and poll until device toggles it.
4649  *
4650  * Return: 0 upon success; < 0 upon failure.
4651  */
4652 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4653 {
4654         int err;
4655         bool flag_res = true;
4656         ktime_t timeout;
4657
4658         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4659                 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4660         if (err) {
4661                 dev_err(hba->dev,
4662                         "%s: setting fDeviceInit flag failed with error %d\n",
4663                         __func__, err);
4664                 goto out;
4665         }
4666
4667         /* Poll fDeviceInit flag to be cleared */
4668         timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4669         do {
4670                 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4671                                         QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4672                 if (!flag_res)
4673                         break;
4674                 usleep_range(500, 1000);
4675         } while (ktime_before(ktime_get(), timeout));
4676
4677         if (err) {
4678                 dev_err(hba->dev,
4679                                 "%s: reading fDeviceInit flag failed with error %d\n",
4680                                 __func__, err);
4681         } else if (flag_res) {
4682                 dev_err(hba->dev,
4683                                 "%s: fDeviceInit was not cleared by the device\n",
4684                                 __func__);
4685                 err = -EBUSY;
4686         }
4687 out:
4688         return err;
4689 }
4690
4691 /**
4692  * ufshcd_make_hba_operational - Make UFS controller operational
4693  * @hba: per adapter instance
4694  *
4695  * To bring UFS host controller to operational state,
4696  * 1. Enable required interrupts
4697  * 2. Configure interrupt aggregation
4698  * 3. Program UTRL and UTMRL base address
4699  * 4. Configure run-stop-registers
4700  *
4701  * Return: 0 on success, non-zero value on failure.
4702  */
4703 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4704 {
4705         int err = 0;
4706         u32 reg;
4707
4708         /* Enable required interrupts */
4709         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4710
4711         /* Configure interrupt aggregation */
4712         if (ufshcd_is_intr_aggr_allowed(hba))
4713                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4714         else
4715                 ufshcd_disable_intr_aggr(hba);
4716
4717         /* Configure UTRL and UTMRL base address registers */
4718         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4719                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4720         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4721                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4722         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4723                         REG_UTP_TASK_REQ_LIST_BASE_L);
4724         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4725                         REG_UTP_TASK_REQ_LIST_BASE_H);
4726
4727         /*
4728          * Make sure base address and interrupt setup are updated before
4729          * enabling the run/stop registers below.
4730          */
4731         wmb();
4732
4733         /*
4734          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4735          */
4736         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4737         if (!(ufshcd_get_lists_status(reg))) {
4738                 ufshcd_enable_run_stop_reg(hba);
4739         } else {
4740                 dev_err(hba->dev,
4741                         "Host controller not ready to process requests");
4742                 err = -EIO;
4743         }
4744
4745         return err;
4746 }
4747 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4748
4749 /**
4750  * ufshcd_hba_stop - Send controller to reset state
4751  * @hba: per adapter instance
4752  */
4753 void ufshcd_hba_stop(struct ufs_hba *hba)
4754 {
4755         unsigned long flags;
4756         int err;
4757
4758         /*
4759          * Obtain the host lock to prevent that the controller is disabled
4760          * while the UFS interrupt handler is active on another CPU.
4761          */
4762         spin_lock_irqsave(hba->host->host_lock, flags);
4763         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4764         spin_unlock_irqrestore(hba->host->host_lock, flags);
4765
4766         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4767                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4768                                         10, 1);
4769         if (err)
4770                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4771 }
4772 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4773
4774 /**
4775  * ufshcd_hba_execute_hce - initialize the controller
4776  * @hba: per adapter instance
4777  *
4778  * The controller resets itself and controller firmware initialization
4779  * sequence kicks off. When controller is ready it will set
4780  * the Host Controller Enable bit to 1.
4781  *
4782  * Return: 0 on success, non-zero value on failure.
4783  */
4784 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4785 {
4786         int retry_outer = 3;
4787         int retry_inner;
4788
4789 start:
4790         if (ufshcd_is_hba_active(hba))
4791                 /* change controller state to "reset state" */
4792                 ufshcd_hba_stop(hba);
4793
4794         /* UniPro link is disabled at this point */
4795         ufshcd_set_link_off(hba);
4796
4797         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4798
4799         /* start controller initialization sequence */
4800         ufshcd_hba_start(hba);
4801
4802         /*
4803          * To initialize a UFS host controller HCE bit must be set to 1.
4804          * During initialization the HCE bit value changes from 1->0->1.
4805          * When the host controller completes initialization sequence
4806          * it sets the value of HCE bit to 1. The same HCE bit is read back
4807          * to check if the controller has completed initialization sequence.
4808          * So without this delay the value HCE = 1, set in the previous
4809          * instruction might be read back.
4810          * This delay can be changed based on the controller.
4811          */
4812         ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4813
4814         /* wait for the host controller to complete initialization */
4815         retry_inner = 50;
4816         while (!ufshcd_is_hba_active(hba)) {
4817                 if (retry_inner) {
4818                         retry_inner--;
4819                 } else {
4820                         dev_err(hba->dev,
4821                                 "Controller enable failed\n");
4822                         if (retry_outer) {
4823                                 retry_outer--;
4824                                 goto start;
4825                         }
4826                         return -EIO;
4827                 }
4828                 usleep_range(1000, 1100);
4829         }
4830
4831         /* enable UIC related interrupts */
4832         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4833
4834         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4835
4836         return 0;
4837 }
4838
4839 int ufshcd_hba_enable(struct ufs_hba *hba)
4840 {
4841         int ret;
4842
4843         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4844                 ufshcd_set_link_off(hba);
4845                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4846
4847                 /* enable UIC related interrupts */
4848                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4849                 ret = ufshcd_dme_reset(hba);
4850                 if (ret) {
4851                         dev_err(hba->dev, "DME_RESET failed\n");
4852                         return ret;
4853                 }
4854
4855                 ret = ufshcd_dme_enable(hba);
4856                 if (ret) {
4857                         dev_err(hba->dev, "Enabling DME failed\n");
4858                         return ret;
4859                 }
4860
4861                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4862         } else {
4863                 ret = ufshcd_hba_execute_hce(hba);
4864         }
4865
4866         return ret;
4867 }
4868 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4869
4870 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4871 {
4872         int tx_lanes = 0, i, err = 0;
4873
4874         if (!peer)
4875                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4876                                &tx_lanes);
4877         else
4878                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4879                                     &tx_lanes);
4880         for (i = 0; i < tx_lanes; i++) {
4881                 if (!peer)
4882                         err = ufshcd_dme_set(hba,
4883                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4884                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4885                                         0);
4886                 else
4887                         err = ufshcd_dme_peer_set(hba,
4888                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4889                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4890                                         0);
4891                 if (err) {
4892                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4893                                 __func__, peer, i, err);
4894                         break;
4895                 }
4896         }
4897
4898         return err;
4899 }
4900
4901 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4902 {
4903         return ufshcd_disable_tx_lcc(hba, true);
4904 }
4905
4906 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4907 {
4908         struct ufs_event_hist *e;
4909
4910         if (id >= UFS_EVT_CNT)
4911                 return;
4912
4913         e = &hba->ufs_stats.event[id];
4914         e->val[e->pos] = val;
4915         e->tstamp[e->pos] = local_clock();
4916         e->cnt += 1;
4917         e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4918
4919         ufshcd_vops_event_notify(hba, id, &val);
4920 }
4921 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4922
4923 /**
4924  * ufshcd_link_startup - Initialize unipro link startup
4925  * @hba: per adapter instance
4926  *
4927  * Return: 0 for success, non-zero in case of failure.
4928  */
4929 static int ufshcd_link_startup(struct ufs_hba *hba)
4930 {
4931         int ret;
4932         int retries = DME_LINKSTARTUP_RETRIES;
4933         bool link_startup_again = false;
4934
4935         /*
4936          * If UFS device isn't active then we will have to issue link startup
4937          * 2 times to make sure the device state move to active.
4938          */
4939         if (!ufshcd_is_ufs_dev_active(hba))
4940                 link_startup_again = true;
4941
4942 link_startup:
4943         do {
4944                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4945
4946                 ret = ufshcd_dme_link_startup(hba);
4947
4948                 /* check if device is detected by inter-connect layer */
4949                 if (!ret && !ufshcd_is_device_present(hba)) {
4950                         ufshcd_update_evt_hist(hba,
4951                                                UFS_EVT_LINK_STARTUP_FAIL,
4952                                                0);
4953                         dev_err(hba->dev, "%s: Device not present\n", __func__);
4954                         ret = -ENXIO;
4955                         goto out;
4956                 }
4957
4958                 /*
4959                  * DME link lost indication is only received when link is up,
4960                  * but we can't be sure if the link is up until link startup
4961                  * succeeds. So reset the local Uni-Pro and try again.
4962                  */
4963                 if (ret && retries && ufshcd_hba_enable(hba)) {
4964                         ufshcd_update_evt_hist(hba,
4965                                                UFS_EVT_LINK_STARTUP_FAIL,
4966                                                (u32)ret);
4967                         goto out;
4968                 }
4969         } while (ret && retries--);
4970
4971         if (ret) {
4972                 /* failed to get the link up... retire */
4973                 ufshcd_update_evt_hist(hba,
4974                                        UFS_EVT_LINK_STARTUP_FAIL,
4975                                        (u32)ret);
4976                 goto out;
4977         }
4978
4979         if (link_startup_again) {
4980                 link_startup_again = false;
4981                 retries = DME_LINKSTARTUP_RETRIES;
4982                 goto link_startup;
4983         }
4984
4985         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4986         ufshcd_init_pwr_info(hba);
4987         ufshcd_print_pwr_info(hba);
4988
4989         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4990                 ret = ufshcd_disable_device_tx_lcc(hba);
4991                 if (ret)
4992                         goto out;
4993         }
4994
4995         /* Include any host controller configuration via UIC commands */
4996         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4997         if (ret)
4998                 goto out;
4999
5000         /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
5001         ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5002         ret = ufshcd_make_hba_operational(hba);
5003 out:
5004         if (ret) {
5005                 dev_err(hba->dev, "link startup failed %d\n", ret);
5006                 ufshcd_print_host_state(hba);
5007                 ufshcd_print_pwr_info(hba);
5008                 ufshcd_print_evt_hist(hba);
5009         }
5010         return ret;
5011 }
5012
5013 /**
5014  * ufshcd_verify_dev_init() - Verify device initialization
5015  * @hba: per-adapter instance
5016  *
5017  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
5018  * device Transport Protocol (UTP) layer is ready after a reset.
5019  * If the UTP layer at the device side is not initialized, it may
5020  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
5021  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
5022  *
5023  * Return: 0 upon success; < 0 upon failure.
5024  */
5025 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
5026 {
5027         int err = 0;
5028         int retries;
5029
5030         ufshcd_hold(hba);
5031         mutex_lock(&hba->dev_cmd.lock);
5032         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
5033                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
5034                                           hba->nop_out_timeout);
5035
5036                 if (!err || err == -ETIMEDOUT)
5037                         break;
5038
5039                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
5040         }
5041         mutex_unlock(&hba->dev_cmd.lock);
5042         ufshcd_release(hba);
5043
5044         if (err)
5045                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
5046         return err;
5047 }
5048
5049 /**
5050  * ufshcd_setup_links - associate link b/w device wlun and other luns
5051  * @sdev: pointer to SCSI device
5052  * @hba: pointer to ufs hba
5053  */
5054 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
5055 {
5056         struct device_link *link;
5057
5058         /*
5059          * Device wlun is the supplier & rest of the luns are consumers.
5060          * This ensures that device wlun suspends after all other luns.
5061          */
5062         if (hba->ufs_device_wlun) {
5063                 link = device_link_add(&sdev->sdev_gendev,
5064                                        &hba->ufs_device_wlun->sdev_gendev,
5065                                        DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
5066                 if (!link) {
5067                         dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
5068                                 dev_name(&hba->ufs_device_wlun->sdev_gendev));
5069                         return;
5070                 }
5071                 hba->luns_avail--;
5072                 /* Ignore REPORT_LUN wlun probing */
5073                 if (hba->luns_avail == 1) {
5074                         ufshcd_rpm_put(hba);
5075                         return;
5076                 }
5077         } else {
5078                 /*
5079                  * Device wlun is probed. The assumption is that WLUNs are
5080                  * scanned before other LUNs.
5081                  */
5082                 hba->luns_avail--;
5083         }
5084 }
5085
5086 /**
5087  * ufshcd_lu_init - Initialize the relevant parameters of the LU
5088  * @hba: per-adapter instance
5089  * @sdev: pointer to SCSI device
5090  */
5091 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev)
5092 {
5093         int len = QUERY_DESC_MAX_SIZE;
5094         u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun);
5095         u8 lun_qdepth = hba->nutrs;
5096         u8 *desc_buf;
5097         int ret;
5098
5099         desc_buf = kzalloc(len, GFP_KERNEL);
5100         if (!desc_buf)
5101                 goto set_qdepth;
5102
5103         ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len);
5104         if (ret < 0) {
5105                 if (ret == -EOPNOTSUPP)
5106                         /* If LU doesn't support unit descriptor, its queue depth is set to 1 */
5107                         lun_qdepth = 1;
5108                 kfree(desc_buf);
5109                 goto set_qdepth;
5110         }
5111
5112         if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) {
5113                 /*
5114                  * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will
5115                  * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth
5116                  */
5117                 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs);
5118         }
5119         /*
5120          * According to UFS device specification, the write protection mode is only supported by
5121          * normal LU, not supported by WLUN.
5122          */
5123         if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported &&
5124             !hba->dev_info.is_lu_power_on_wp &&
5125             desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP)
5126                 hba->dev_info.is_lu_power_on_wp = true;
5127
5128         /* In case of RPMB LU, check if advanced RPMB mode is enabled */
5129         if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN &&
5130             desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
5131                 hba->dev_info.b_advanced_rpmb_en = true;
5132
5133
5134         kfree(desc_buf);
5135 set_qdepth:
5136         /*
5137          * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose
5138          * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue.
5139          */
5140         dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth);
5141         scsi_change_queue_depth(sdev, lun_qdepth);
5142 }
5143
5144 /**
5145  * ufshcd_slave_alloc - handle initial SCSI device configurations
5146  * @sdev: pointer to SCSI device
5147  *
5148  * Return: success.
5149  */
5150 static int ufshcd_slave_alloc(struct scsi_device *sdev)
5151 {
5152         struct ufs_hba *hba;
5153
5154         hba = shost_priv(sdev->host);
5155
5156         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5157         sdev->use_10_for_ms = 1;
5158
5159         /* DBD field should be set to 1 in mode sense(10) */
5160         sdev->set_dbd_for_ms = 1;
5161
5162         /* allow SCSI layer to restart the device in case of errors */
5163         sdev->allow_restart = 1;
5164
5165         /* REPORT SUPPORTED OPERATION CODES is not supported */
5166         sdev->no_report_opcodes = 1;
5167
5168         /* WRITE_SAME command is not supported */
5169         sdev->no_write_same = 1;
5170
5171         ufshcd_lu_init(hba, sdev);
5172
5173         ufshcd_setup_links(hba, sdev);
5174
5175         return 0;
5176 }
5177
5178 /**
5179  * ufshcd_change_queue_depth - change queue depth
5180  * @sdev: pointer to SCSI device
5181  * @depth: required depth to set
5182  *
5183  * Change queue depth and make sure the max. limits are not crossed.
5184  *
5185  * Return: new queue depth.
5186  */
5187 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5188 {
5189         return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5190 }
5191
5192 /**
5193  * ufshcd_slave_configure - adjust SCSI device configurations
5194  * @sdev: pointer to SCSI device
5195  *
5196  * Return: 0 (success).
5197  */
5198 static int ufshcd_slave_configure(struct scsi_device *sdev)
5199 {
5200         struct ufs_hba *hba = shost_priv(sdev->host);
5201         struct request_queue *q = sdev->request_queue;
5202
5203         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
5204
5205         /*
5206          * Block runtime-pm until all consumers are added.
5207          * Refer ufshcd_setup_links().
5208          */
5209         if (is_device_wlun(sdev))
5210                 pm_runtime_get_noresume(&sdev->sdev_gendev);
5211         else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5212                 sdev->rpm_autosuspend = 1;
5213         /*
5214          * Do not print messages during runtime PM to avoid never-ending cycles
5215          * of messages written back to storage by user space causing runtime
5216          * resume, causing more messages and so on.
5217          */
5218         sdev->silence_suspend = 1;
5219
5220         if (hba->vops && hba->vops->config_scsi_dev)
5221                 hba->vops->config_scsi_dev(sdev);
5222
5223         ufshcd_crypto_register(hba, q);
5224
5225         return 0;
5226 }
5227
5228 /**
5229  * ufshcd_slave_destroy - remove SCSI device configurations
5230  * @sdev: pointer to SCSI device
5231  */
5232 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5233 {
5234         struct ufs_hba *hba;
5235         unsigned long flags;
5236
5237         hba = shost_priv(sdev->host);
5238
5239         /* Drop the reference as it won't be needed anymore */
5240         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5241                 spin_lock_irqsave(hba->host->host_lock, flags);
5242                 hba->ufs_device_wlun = NULL;
5243                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5244         } else if (hba->ufs_device_wlun) {
5245                 struct device *supplier = NULL;
5246
5247                 /* Ensure UFS Device WLUN exists and does not disappear */
5248                 spin_lock_irqsave(hba->host->host_lock, flags);
5249                 if (hba->ufs_device_wlun) {
5250                         supplier = &hba->ufs_device_wlun->sdev_gendev;
5251                         get_device(supplier);
5252                 }
5253                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5254
5255                 if (supplier) {
5256                         /*
5257                          * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5258                          * device will not have been registered but can still
5259                          * have a device link holding a reference to the device.
5260                          */
5261                         device_link_remove(&sdev->sdev_gendev, supplier);
5262                         put_device(supplier);
5263                 }
5264         }
5265 }
5266
5267 /**
5268  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5269  * @lrbp: pointer to local reference block of completed command
5270  * @scsi_status: SCSI command status
5271  *
5272  * Return: value base on SCSI command status.
5273  */
5274 static inline int
5275 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5276 {
5277         int result = 0;
5278
5279         switch (scsi_status) {
5280         case SAM_STAT_CHECK_CONDITION:
5281                 ufshcd_copy_sense_data(lrbp);
5282                 fallthrough;
5283         case SAM_STAT_GOOD:
5284                 result |= DID_OK << 16 | scsi_status;
5285                 break;
5286         case SAM_STAT_TASK_SET_FULL:
5287         case SAM_STAT_BUSY:
5288         case SAM_STAT_TASK_ABORTED:
5289                 ufshcd_copy_sense_data(lrbp);
5290                 result |= scsi_status;
5291                 break;
5292         default:
5293                 result |= DID_ERROR << 16;
5294                 break;
5295         } /* end of switch */
5296
5297         return result;
5298 }
5299
5300 /**
5301  * ufshcd_transfer_rsp_status - Get overall status of the response
5302  * @hba: per adapter instance
5303  * @lrbp: pointer to local reference block of completed command
5304  * @cqe: pointer to the completion queue entry
5305  *
5306  * Return: result of the command to notify SCSI midlayer.
5307  */
5308 static inline int
5309 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
5310                            struct cq_entry *cqe)
5311 {
5312         int result = 0;
5313         int scsi_status;
5314         enum utp_ocs ocs;
5315         u8 upiu_flags;
5316         u32 resid;
5317
5318         upiu_flags = lrbp->ucd_rsp_ptr->header.flags;
5319         resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count);
5320         /*
5321          * Test !overflow instead of underflow to support UFS devices that do
5322          * not set either flag.
5323          */
5324         if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
5325                 scsi_set_resid(lrbp->cmd, resid);
5326
5327         /* overall command status of utrd */
5328         ocs = ufshcd_get_tr_ocs(lrbp, cqe);
5329
5330         if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5331                 if (lrbp->ucd_rsp_ptr->header.response ||
5332                     lrbp->ucd_rsp_ptr->header.status)
5333                         ocs = OCS_SUCCESS;
5334         }
5335
5336         switch (ocs) {
5337         case OCS_SUCCESS:
5338                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5339                 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) {
5340                 case UPIU_TRANSACTION_RESPONSE:
5341                         /*
5342                          * get the result based on SCSI status response
5343                          * to notify the SCSI midlayer of the command status
5344                          */
5345                         scsi_status = lrbp->ucd_rsp_ptr->header.status;
5346                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5347
5348                         /*
5349                          * Currently we are only supporting BKOPs exception
5350                          * events hence we can ignore BKOPs exception event
5351                          * during power management callbacks. BKOPs exception
5352                          * event is not expected to be raised in runtime suspend
5353                          * callback as it allows the urgent bkops.
5354                          * During system suspend, we are anyway forcefully
5355                          * disabling the bkops and if urgent bkops is needed
5356                          * it will be enabled on system resume. Long term
5357                          * solution could be to abort the system suspend if
5358                          * UFS device needs urgent BKOPs.
5359                          */
5360                         if (!hba->pm_op_in_progress &&
5361                             !ufshcd_eh_in_progress(hba) &&
5362                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5363                                 /* Flushed in suspend */
5364                                 schedule_work(&hba->eeh_work);
5365                         break;
5366                 case UPIU_TRANSACTION_REJECT_UPIU:
5367                         /* TODO: handle Reject UPIU Response */
5368                         result = DID_ERROR << 16;
5369                         dev_err(hba->dev,
5370                                 "Reject UPIU not fully implemented\n");
5371                         break;
5372                 default:
5373                         dev_err(hba->dev,
5374                                 "Unexpected request response code = %x\n",
5375                                 result);
5376                         result = DID_ERROR << 16;
5377                         break;
5378                 }
5379                 break;
5380         case OCS_ABORTED:
5381                 result |= DID_ABORT << 16;
5382                 break;
5383         case OCS_INVALID_COMMAND_STATUS:
5384                 result |= DID_REQUEUE << 16;
5385                 break;
5386         case OCS_INVALID_CMD_TABLE_ATTR:
5387         case OCS_INVALID_PRDT_ATTR:
5388         case OCS_MISMATCH_DATA_BUF_SIZE:
5389         case OCS_MISMATCH_RESP_UPIU_SIZE:
5390         case OCS_PEER_COMM_FAILURE:
5391         case OCS_FATAL_ERROR:
5392         case OCS_DEVICE_FATAL_ERROR:
5393         case OCS_INVALID_CRYPTO_CONFIG:
5394         case OCS_GENERAL_CRYPTO_ERROR:
5395         default:
5396                 result |= DID_ERROR << 16;
5397                 dev_err(hba->dev,
5398                                 "OCS error from controller = %x for tag %d\n",
5399                                 ocs, lrbp->task_tag);
5400                 ufshcd_print_evt_hist(hba);
5401                 ufshcd_print_host_state(hba);
5402                 break;
5403         } /* end of switch */
5404
5405         if ((host_byte(result) != DID_OK) &&
5406             (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5407                 ufshcd_print_tr(hba, lrbp->task_tag, true);
5408         return result;
5409 }
5410
5411 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5412                                          u32 intr_mask)
5413 {
5414         if (!ufshcd_is_auto_hibern8_supported(hba) ||
5415             !ufshcd_is_auto_hibern8_enabled(hba))
5416                 return false;
5417
5418         if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5419                 return false;
5420
5421         if (hba->active_uic_cmd &&
5422             (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5423             hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5424                 return false;
5425
5426         return true;
5427 }
5428
5429 /**
5430  * ufshcd_uic_cmd_compl - handle completion of uic command
5431  * @hba: per adapter instance
5432  * @intr_status: interrupt status generated by the controller
5433  *
5434  * Return:
5435  *  IRQ_HANDLED - If interrupt is valid
5436  *  IRQ_NONE    - If invalid interrupt
5437  */
5438 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5439 {
5440         irqreturn_t retval = IRQ_NONE;
5441
5442         spin_lock(hba->host->host_lock);
5443         if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5444                 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5445
5446         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5447                 hba->active_uic_cmd->argument2 |=
5448                         ufshcd_get_uic_cmd_result(hba);
5449                 hba->active_uic_cmd->argument3 =
5450                         ufshcd_get_dme_attr_val(hba);
5451                 if (!hba->uic_async_done)
5452                         hba->active_uic_cmd->cmd_active = 0;
5453                 complete(&hba->active_uic_cmd->done);
5454                 retval = IRQ_HANDLED;
5455         }
5456
5457         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5458                 hba->active_uic_cmd->cmd_active = 0;
5459                 complete(hba->uic_async_done);
5460                 retval = IRQ_HANDLED;
5461         }
5462
5463         if (retval == IRQ_HANDLED)
5464                 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5465                                              UFS_CMD_COMP);
5466         spin_unlock(hba->host->host_lock);
5467         return retval;
5468 }
5469
5470 /* Release the resources allocated for processing a SCSI command. */
5471 void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5472                              struct ufshcd_lrb *lrbp)
5473 {
5474         struct scsi_cmnd *cmd = lrbp->cmd;
5475
5476         scsi_dma_unmap(cmd);
5477         ufshcd_release(hba);
5478         ufshcd_clk_scaling_update_busy(hba);
5479 }
5480
5481 /**
5482  * ufshcd_compl_one_cqe - handle a completion queue entry
5483  * @hba: per adapter instance
5484  * @task_tag: the task tag of the request to be completed
5485  * @cqe: pointer to the completion queue entry
5486  */
5487 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
5488                           struct cq_entry *cqe)
5489 {
5490         struct ufshcd_lrb *lrbp;
5491         struct scsi_cmnd *cmd;
5492         enum utp_ocs ocs;
5493
5494         lrbp = &hba->lrb[task_tag];
5495         lrbp->compl_time_stamp = ktime_get();
5496         cmd = lrbp->cmd;
5497         if (cmd) {
5498                 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5499                         ufshcd_update_monitor(hba, lrbp);
5500                 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP);
5501                 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
5502                 ufshcd_release_scsi_cmd(hba, lrbp);
5503                 /* Do not touch lrbp after scsi done */
5504                 scsi_done(cmd);
5505         } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5506                    lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5507                 if (hba->dev_cmd.complete) {
5508                         if (cqe) {
5509                                 ocs = le32_to_cpu(cqe->status) & MASK_OCS;
5510                                 lrbp->utr_descriptor_ptr->header.ocs = ocs;
5511                         }
5512                         complete(hba->dev_cmd.complete);
5513                 }
5514         }
5515 }
5516
5517 /**
5518  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5519  * @hba: per adapter instance
5520  * @completed_reqs: bitmask that indicates which requests to complete
5521  */
5522 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5523                                         unsigned long completed_reqs)
5524 {
5525         int tag;
5526
5527         for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5528                 ufshcd_compl_one_cqe(hba, tag, NULL);
5529 }
5530
5531 /* Any value that is not an existing queue number is fine for this constant. */
5532 enum {
5533         UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5534 };
5535
5536 static void ufshcd_clear_polled(struct ufs_hba *hba,
5537                                 unsigned long *completed_reqs)
5538 {
5539         int tag;
5540
5541         for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5542                 struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5543
5544                 if (!cmd)
5545                         continue;
5546                 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5547                         __clear_bit(tag, completed_reqs);
5548         }
5549 }
5550
5551 /*
5552  * Return: > 0 if one or more commands have been completed or 0 if no
5553  * requests have been completed.
5554  */
5555 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5556 {
5557         struct ufs_hba *hba = shost_priv(shost);
5558         unsigned long completed_reqs, flags;
5559         u32 tr_doorbell;
5560         struct ufs_hw_queue *hwq;
5561
5562         if (is_mcq_enabled(hba)) {
5563                 hwq = &hba->uhq[queue_num];
5564
5565                 return ufshcd_mcq_poll_cqe_lock(hba, hwq);
5566         }
5567
5568         spin_lock_irqsave(&hba->outstanding_lock, flags);
5569         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5570         completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5571         WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5572                   "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5573                   hba->outstanding_reqs);
5574         if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) {
5575                 /* Do not complete polled requests from interrupt context. */
5576                 ufshcd_clear_polled(hba, &completed_reqs);
5577         }
5578         hba->outstanding_reqs &= ~completed_reqs;
5579         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5580
5581         if (completed_reqs)
5582                 __ufshcd_transfer_req_compl(hba, completed_reqs);
5583
5584         return completed_reqs != 0;
5585 }
5586
5587 /**
5588  * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
5589  * invoked from the error handler context or ufshcd_host_reset_and_restore()
5590  * to complete the pending transfers and free the resources associated with
5591  * the scsi command.
5592  *
5593  * @hba: per adapter instance
5594  * @force_compl: This flag is set to true when invoked
5595  * from ufshcd_host_reset_and_restore() in which case it requires special
5596  * handling because the host controller has been reset by ufshcd_hba_stop().
5597  */
5598 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
5599                                               bool force_compl)
5600 {
5601         struct ufs_hw_queue *hwq;
5602         struct ufshcd_lrb *lrbp;
5603         struct scsi_cmnd *cmd;
5604         unsigned long flags;
5605         u32 hwq_num, utag;
5606         int tag;
5607
5608         for (tag = 0; tag < hba->nutrs; tag++) {
5609                 lrbp = &hba->lrb[tag];
5610                 cmd = lrbp->cmd;
5611                 if (!ufshcd_cmd_inflight(cmd) ||
5612                     test_bit(SCMD_STATE_COMPLETE, &cmd->state))
5613                         continue;
5614
5615                 utag = blk_mq_unique_tag(scsi_cmd_to_rq(cmd));
5616                 hwq_num = blk_mq_unique_tag_to_hwq(utag);
5617                 hwq = &hba->uhq[hwq_num];
5618
5619                 if (force_compl) {
5620                         ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
5621                         /*
5622                          * For those cmds of which the cqes are not present
5623                          * in the cq, complete them explicitly.
5624                          */
5625                         spin_lock_irqsave(&hwq->cq_lock, flags);
5626                         if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
5627                                 set_host_byte(cmd, DID_REQUEUE);
5628                                 ufshcd_release_scsi_cmd(hba, lrbp);
5629                                 scsi_done(cmd);
5630                         }
5631                         spin_unlock_irqrestore(&hwq->cq_lock, flags);
5632                 } else {
5633                         ufshcd_mcq_poll_cqe_lock(hba, hwq);
5634                 }
5635         }
5636 }
5637
5638 /**
5639  * ufshcd_transfer_req_compl - handle SCSI and query command completion
5640  * @hba: per adapter instance
5641  *
5642  * Return:
5643  *  IRQ_HANDLED - If interrupt is valid
5644  *  IRQ_NONE    - If invalid interrupt
5645  */
5646 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5647 {
5648         /* Resetting interrupt aggregation counters first and reading the
5649          * DOOR_BELL afterward allows us to handle all the completed requests.
5650          * In order to prevent other interrupts starvation the DB is read once
5651          * after reset. The down side of this solution is the possibility of
5652          * false interrupt if device completes another request after resetting
5653          * aggregation and before reading the DB.
5654          */
5655         if (ufshcd_is_intr_aggr_allowed(hba) &&
5656             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5657                 ufshcd_reset_intr_aggr(hba);
5658
5659         if (ufs_fail_completion(hba))
5660                 return IRQ_HANDLED;
5661
5662         /*
5663          * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5664          * do not want polling to trigger spurious interrupt complaints.
5665          */
5666         ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5667
5668         return IRQ_HANDLED;
5669 }
5670
5671 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5672 {
5673         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5674                                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5675                                        &ee_ctrl_mask);
5676 }
5677
5678 int ufshcd_write_ee_control(struct ufs_hba *hba)
5679 {
5680         int err;
5681
5682         mutex_lock(&hba->ee_ctrl_mutex);
5683         err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5684         mutex_unlock(&hba->ee_ctrl_mutex);
5685         if (err)
5686                 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5687                         __func__, err);
5688         return err;
5689 }
5690
5691 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
5692                              const u16 *other_mask, u16 set, u16 clr)
5693 {
5694         u16 new_mask, ee_ctrl_mask;
5695         int err = 0;
5696
5697         mutex_lock(&hba->ee_ctrl_mutex);
5698         new_mask = (*mask & ~clr) | set;
5699         ee_ctrl_mask = new_mask | *other_mask;
5700         if (ee_ctrl_mask != hba->ee_ctrl_mask)
5701                 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5702         /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5703         if (!err) {
5704                 hba->ee_ctrl_mask = ee_ctrl_mask;
5705                 *mask = new_mask;
5706         }
5707         mutex_unlock(&hba->ee_ctrl_mutex);
5708         return err;
5709 }
5710
5711 /**
5712  * ufshcd_disable_ee - disable exception event
5713  * @hba: per-adapter instance
5714  * @mask: exception event to disable
5715  *
5716  * Disables exception event in the device so that the EVENT_ALERT
5717  * bit is not set.
5718  *
5719  * Return: zero on success, non-zero error value on failure.
5720  */
5721 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5722 {
5723         return ufshcd_update_ee_drv_mask(hba, 0, mask);
5724 }
5725
5726 /**
5727  * ufshcd_enable_ee - enable exception event
5728  * @hba: per-adapter instance
5729  * @mask: exception event to enable
5730  *
5731  * Enable corresponding exception event in the device to allow
5732  * device to alert host in critical scenarios.
5733  *
5734  * Return: zero on success, non-zero error value on failure.
5735  */
5736 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5737 {
5738         return ufshcd_update_ee_drv_mask(hba, mask, 0);
5739 }
5740
5741 /**
5742  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5743  * @hba: per-adapter instance
5744  *
5745  * Allow device to manage background operations on its own. Enabling
5746  * this might lead to inconsistent latencies during normal data transfers
5747  * as the device is allowed to manage its own way of handling background
5748  * operations.
5749  *
5750  * Return: zero on success, non-zero on failure.
5751  */
5752 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5753 {
5754         int err = 0;
5755
5756         if (hba->auto_bkops_enabled)
5757                 goto out;
5758
5759         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5760                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5761         if (err) {
5762                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5763                                 __func__, err);
5764                 goto out;
5765         }
5766
5767         hba->auto_bkops_enabled = true;
5768         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5769
5770         /* No need of URGENT_BKOPS exception from the device */
5771         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5772         if (err)
5773                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5774                                 __func__, err);
5775 out:
5776         return err;
5777 }
5778
5779 /**
5780  * ufshcd_disable_auto_bkops - block device in doing background operations
5781  * @hba: per-adapter instance
5782  *
5783  * Disabling background operations improves command response latency but
5784  * has drawback of device moving into critical state where the device is
5785  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5786  * host is idle so that BKOPS are managed effectively without any negative
5787  * impacts.
5788  *
5789  * Return: zero on success, non-zero on failure.
5790  */
5791 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5792 {
5793         int err = 0;
5794
5795         if (!hba->auto_bkops_enabled)
5796                 goto out;
5797
5798         /*
5799          * If host assisted BKOPs is to be enabled, make sure
5800          * urgent bkops exception is allowed.
5801          */
5802         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5803         if (err) {
5804                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5805                                 __func__, err);
5806                 goto out;
5807         }
5808
5809         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5810                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5811         if (err) {
5812                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5813                                 __func__, err);
5814                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5815                 goto out;
5816         }
5817
5818         hba->auto_bkops_enabled = false;
5819         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5820         hba->is_urgent_bkops_lvl_checked = false;
5821 out:
5822         return err;
5823 }
5824
5825 /**
5826  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5827  * @hba: per adapter instance
5828  *
5829  * After a device reset the device may toggle the BKOPS_EN flag
5830  * to default value. The s/w tracking variables should be updated
5831  * as well. This function would change the auto-bkops state based on
5832  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5833  */
5834 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5835 {
5836         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5837                 hba->auto_bkops_enabled = false;
5838                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5839                 ufshcd_enable_auto_bkops(hba);
5840         } else {
5841                 hba->auto_bkops_enabled = true;
5842                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5843                 ufshcd_disable_auto_bkops(hba);
5844         }
5845         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5846         hba->is_urgent_bkops_lvl_checked = false;
5847 }
5848
5849 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5850 {
5851         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5852                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5853 }
5854
5855 /**
5856  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5857  * @hba: per-adapter instance
5858  * @status: bkops_status value
5859  *
5860  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5861  * flag in the device to permit background operations if the device
5862  * bkops_status is greater than or equal to "status" argument passed to
5863  * this function, disable otherwise.
5864  *
5865  * Return: 0 for success, non-zero in case of failure.
5866  *
5867  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5868  * to know whether auto bkops is enabled or disabled after this function
5869  * returns control to it.
5870  */
5871 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5872                              enum bkops_status status)
5873 {
5874         int err;
5875         u32 curr_status = 0;
5876
5877         err = ufshcd_get_bkops_status(hba, &curr_status);
5878         if (err) {
5879                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5880                                 __func__, err);
5881                 goto out;
5882         } else if (curr_status > BKOPS_STATUS_MAX) {
5883                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5884                                 __func__, curr_status);
5885                 err = -EINVAL;
5886                 goto out;
5887         }
5888
5889         if (curr_status >= status)
5890                 err = ufshcd_enable_auto_bkops(hba);
5891         else
5892                 err = ufshcd_disable_auto_bkops(hba);
5893 out:
5894         return err;
5895 }
5896
5897 /**
5898  * ufshcd_urgent_bkops - handle urgent bkops exception event
5899  * @hba: per-adapter instance
5900  *
5901  * Enable fBackgroundOpsEn flag in the device to permit background
5902  * operations.
5903  *
5904  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5905  * and negative error value for any other failure.
5906  *
5907  * Return: 0 upon success; < 0 upon failure.
5908  */
5909 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5910 {
5911         return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5912 }
5913
5914 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5915 {
5916         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5917                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5918 }
5919
5920 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5921 {
5922         int err;
5923         u32 curr_status = 0;
5924
5925         if (hba->is_urgent_bkops_lvl_checked)
5926                 goto enable_auto_bkops;
5927
5928         err = ufshcd_get_bkops_status(hba, &curr_status);
5929         if (err) {
5930                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5931                                 __func__, err);
5932                 goto out;
5933         }
5934
5935         /*
5936          * We are seeing that some devices are raising the urgent bkops
5937          * exception events even when BKOPS status doesn't indicate performace
5938          * impacted or critical. Handle these device by determining their urgent
5939          * bkops status at runtime.
5940          */
5941         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5942                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5943                                 __func__, curr_status);
5944                 /* update the current status as the urgent bkops level */
5945                 hba->urgent_bkops_lvl = curr_status;
5946                 hba->is_urgent_bkops_lvl_checked = true;
5947         }
5948
5949 enable_auto_bkops:
5950         err = ufshcd_enable_auto_bkops(hba);
5951 out:
5952         if (err < 0)
5953                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5954                                 __func__, err);
5955 }
5956
5957 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5958 {
5959         u32 value;
5960
5961         if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5962                                 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5963                 return;
5964
5965         dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5966
5967         ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5968
5969         /*
5970          * A placeholder for the platform vendors to add whatever additional
5971          * steps required
5972          */
5973 }
5974
5975 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5976 {
5977         u8 index;
5978         enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5979                                    UPIU_QUERY_OPCODE_CLEAR_FLAG;
5980
5981         index = ufshcd_wb_get_query_index(hba);
5982         return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
5983 }
5984
5985 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
5986 {
5987         int ret;
5988
5989         if (!ufshcd_is_wb_allowed(hba) ||
5990             hba->dev_info.wb_enabled == enable)
5991                 return 0;
5992
5993         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
5994         if (ret) {
5995                 dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
5996                         __func__, enable ? "enabling" : "disabling", ret);
5997                 return ret;
5998         }
5999
6000         hba->dev_info.wb_enabled = enable;
6001         dev_dbg(hba->dev, "%s: Write Booster %s\n",
6002                         __func__, enable ? "enabled" : "disabled");
6003
6004         return ret;
6005 }
6006
6007 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
6008                                                  bool enable)
6009 {
6010         int ret;
6011
6012         ret = __ufshcd_wb_toggle(hba, enable,
6013                         QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
6014         if (ret) {
6015                 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
6016                         __func__, enable ? "enabling" : "disabling", ret);
6017                 return;
6018         }
6019         dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
6020                         __func__, enable ? "enabled" : "disabled");
6021 }
6022
6023 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable)
6024 {
6025         int ret;
6026
6027         if (!ufshcd_is_wb_allowed(hba) ||
6028             hba->dev_info.wb_buf_flush_enabled == enable)
6029                 return 0;
6030
6031         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
6032         if (ret) {
6033                 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
6034                         __func__, enable ? "enabling" : "disabling", ret);
6035                 return ret;
6036         }
6037
6038         hba->dev_info.wb_buf_flush_enabled = enable;
6039         dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
6040                         __func__, enable ? "enabled" : "disabled");
6041
6042         return ret;
6043 }
6044
6045 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
6046                                                 u32 avail_buf)
6047 {
6048         u32 cur_buf;
6049         int ret;
6050         u8 index;
6051
6052         index = ufshcd_wb_get_query_index(hba);
6053         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6054                                               QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
6055                                               index, 0, &cur_buf);
6056         if (ret) {
6057                 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
6058                         __func__, ret);
6059                 return false;
6060         }
6061
6062         if (!cur_buf) {
6063                 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
6064                          cur_buf);
6065                 return false;
6066         }
6067         /* Let it continue to flush when available buffer exceeds threshold */
6068         return avail_buf < hba->vps->wb_flush_threshold;
6069 }
6070
6071 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
6072 {
6073         if (ufshcd_is_wb_buf_flush_allowed(hba))
6074                 ufshcd_wb_toggle_buf_flush(hba, false);
6075
6076         ufshcd_wb_toggle_buf_flush_during_h8(hba, false);
6077         ufshcd_wb_toggle(hba, false);
6078         hba->caps &= ~UFSHCD_CAP_WB_EN;
6079
6080         dev_info(hba->dev, "%s: WB force disabled\n", __func__);
6081 }
6082
6083 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
6084 {
6085         u32 lifetime;
6086         int ret;
6087         u8 index;
6088
6089         index = ufshcd_wb_get_query_index(hba);
6090         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6091                                       QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
6092                                       index, 0, &lifetime);
6093         if (ret) {
6094                 dev_err(hba->dev,
6095                         "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
6096                         __func__, ret);
6097                 return false;
6098         }
6099
6100         if (lifetime == UFS_WB_EXCEED_LIFETIME) {
6101                 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
6102                         __func__, lifetime);
6103                 return false;
6104         }
6105
6106         dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
6107                 __func__, lifetime);
6108
6109         return true;
6110 }
6111
6112 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
6113 {
6114         int ret;
6115         u32 avail_buf;
6116         u8 index;
6117
6118         if (!ufshcd_is_wb_allowed(hba))
6119                 return false;
6120
6121         if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
6122                 ufshcd_wb_force_disable(hba);
6123                 return false;
6124         }
6125
6126         /*
6127          * The ufs device needs the vcc to be ON to flush.
6128          * With user-space reduction enabled, it's enough to enable flush
6129          * by checking only the available buffer. The threshold
6130          * defined here is > 90% full.
6131          * With user-space preserved enabled, the current-buffer
6132          * should be checked too because the wb buffer size can reduce
6133          * when disk tends to be full. This info is provided by current
6134          * buffer (dCurrentWriteBoosterBufferSize). There's no point in
6135          * keeping vcc on when current buffer is empty.
6136          */
6137         index = ufshcd_wb_get_query_index(hba);
6138         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6139                                       QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
6140                                       index, 0, &avail_buf);
6141         if (ret) {
6142                 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
6143                          __func__, ret);
6144                 return false;
6145         }
6146
6147         if (!hba->dev_info.b_presrv_uspc_en)
6148                 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
6149
6150         return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
6151 }
6152
6153 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
6154 {
6155         struct ufs_hba *hba = container_of(to_delayed_work(work),
6156                                            struct ufs_hba,
6157                                            rpm_dev_flush_recheck_work);
6158         /*
6159          * To prevent unnecessary VCC power drain after device finishes
6160          * WriteBooster buffer flush or Auto BKOPs, force runtime resume
6161          * after a certain delay to recheck the threshold by next runtime
6162          * suspend.
6163          */
6164         ufshcd_rpm_get_sync(hba);
6165         ufshcd_rpm_put_sync(hba);
6166 }
6167
6168 /**
6169  * ufshcd_exception_event_handler - handle exceptions raised by device
6170  * @work: pointer to work data
6171  *
6172  * Read bExceptionEventStatus attribute from the device and handle the
6173  * exception event accordingly.
6174  */
6175 static void ufshcd_exception_event_handler(struct work_struct *work)
6176 {
6177         struct ufs_hba *hba;
6178         int err;
6179         u32 status = 0;
6180         hba = container_of(work, struct ufs_hba, eeh_work);
6181
6182         ufshcd_scsi_block_requests(hba);
6183         err = ufshcd_get_ee_status(hba, &status);
6184         if (err) {
6185                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
6186                                 __func__, err);
6187                 goto out;
6188         }
6189
6190         trace_ufshcd_exception_event(dev_name(hba->dev), status);
6191
6192         if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
6193                 ufshcd_bkops_exception_event_handler(hba);
6194
6195         if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
6196                 ufshcd_temp_exception_event_handler(hba, status);
6197
6198         ufs_debugfs_exception_event(hba, status);
6199 out:
6200         ufshcd_scsi_unblock_requests(hba);
6201 }
6202
6203 /* Complete requests that have door-bell cleared */
6204 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl)
6205 {
6206         if (is_mcq_enabled(hba))
6207                 ufshcd_mcq_compl_pending_transfer(hba, force_compl);
6208         else
6209                 ufshcd_transfer_req_compl(hba);
6210
6211         ufshcd_tmc_handler(hba);
6212 }
6213
6214 /**
6215  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6216  *                              to recover from the DL NAC errors or not.
6217  * @hba: per-adapter instance
6218  *
6219  * Return: true if error handling is required, false otherwise.
6220  */
6221 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
6222 {
6223         unsigned long flags;
6224         bool err_handling = true;
6225
6226         spin_lock_irqsave(hba->host->host_lock, flags);
6227         /*
6228          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
6229          * device fatal error and/or DL NAC & REPLAY timeout errors.
6230          */
6231         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6232                 goto out;
6233
6234         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6235             ((hba->saved_err & UIC_ERROR) &&
6236              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6237                 goto out;
6238
6239         if ((hba->saved_err & UIC_ERROR) &&
6240             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6241                 int err;
6242                 /*
6243                  * wait for 50ms to see if we can get any other errors or not.
6244                  */
6245                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6246                 msleep(50);
6247                 spin_lock_irqsave(hba->host->host_lock, flags);
6248
6249                 /*
6250                  * now check if we have got any other severe errors other than
6251                  * DL NAC error?
6252                  */
6253                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
6254                     ((hba->saved_err & UIC_ERROR) &&
6255                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6256                         goto out;
6257
6258                 /*
6259                  * As DL NAC is the only error received so far, send out NOP
6260                  * command to confirm if link is still active or not.
6261                  *   - If we don't get any response then do error recovery.
6262                  *   - If we get response then clear the DL NAC error bit.
6263                  */
6264
6265                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6266                 err = ufshcd_verify_dev_init(hba);
6267                 spin_lock_irqsave(hba->host->host_lock, flags);
6268
6269                 if (err)
6270                         goto out;
6271
6272                 /* Link seems to be alive hence ignore the DL NAC errors */
6273                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6274                         hba->saved_err &= ~UIC_ERROR;
6275                 /* clear NAC error */
6276                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6277                 if (!hba->saved_uic_err)
6278                         err_handling = false;
6279         }
6280 out:
6281         spin_unlock_irqrestore(hba->host->host_lock, flags);
6282         return err_handling;
6283 }
6284
6285 /* host lock must be held before calling this func */
6286 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6287 {
6288         return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6289                (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6290 }
6291
6292 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6293 {
6294         lockdep_assert_held(hba->host->host_lock);
6295
6296         /* handle fatal errors only when link is not in error state */
6297         if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6298                 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6299                     ufshcd_is_saved_err_fatal(hba))
6300                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6301                 else
6302                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6303                 queue_work(hba->eh_wq, &hba->eh_work);
6304         }
6305 }
6306
6307 static void ufshcd_force_error_recovery(struct ufs_hba *hba)
6308 {
6309         spin_lock_irq(hba->host->host_lock);
6310         hba->force_reset = true;
6311         ufshcd_schedule_eh_work(hba);
6312         spin_unlock_irq(hba->host->host_lock);
6313 }
6314
6315 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6316 {
6317         mutex_lock(&hba->wb_mutex);
6318         down_write(&hba->clk_scaling_lock);
6319         hba->clk_scaling.is_allowed = allow;
6320         up_write(&hba->clk_scaling_lock);
6321         mutex_unlock(&hba->wb_mutex);
6322 }
6323
6324 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6325 {
6326         if (suspend) {
6327                 if (hba->clk_scaling.is_enabled)
6328                         ufshcd_suspend_clkscaling(hba);
6329                 ufshcd_clk_scaling_allow(hba, false);
6330         } else {
6331                 ufshcd_clk_scaling_allow(hba, true);
6332                 if (hba->clk_scaling.is_enabled)
6333                         ufshcd_resume_clkscaling(hba);
6334         }
6335 }
6336
6337 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6338 {
6339         ufshcd_rpm_get_sync(hba);
6340         if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6341             hba->is_sys_suspended) {
6342                 enum ufs_pm_op pm_op;
6343
6344                 /*
6345                  * Don't assume anything of resume, if
6346                  * resume fails, irq and clocks can be OFF, and powers
6347                  * can be OFF or in LPM.
6348                  */
6349                 ufshcd_setup_hba_vreg(hba, true);
6350                 ufshcd_enable_irq(hba);
6351                 ufshcd_setup_vreg(hba, true);
6352                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6353                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6354                 ufshcd_hold(hba);
6355                 if (!ufshcd_is_clkgating_allowed(hba))
6356                         ufshcd_setup_clocks(hba, true);
6357                 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6358                 ufshcd_vops_resume(hba, pm_op);
6359         } else {
6360                 ufshcd_hold(hba);
6361                 if (ufshcd_is_clkscaling_supported(hba) &&
6362                     hba->clk_scaling.is_enabled)
6363                         ufshcd_suspend_clkscaling(hba);
6364                 ufshcd_clk_scaling_allow(hba, false);
6365         }
6366         ufshcd_scsi_block_requests(hba);
6367         /* Wait for ongoing ufshcd_queuecommand() calls to finish. */
6368         blk_mq_wait_quiesce_done(&hba->host->tag_set);
6369         cancel_work_sync(&hba->eeh_work);
6370 }
6371
6372 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6373 {
6374         ufshcd_scsi_unblock_requests(hba);
6375         ufshcd_release(hba);
6376         if (ufshcd_is_clkscaling_supported(hba))
6377                 ufshcd_clk_scaling_suspend(hba, false);
6378         ufshcd_rpm_put(hba);
6379 }
6380
6381 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6382 {
6383         return (!hba->is_powered || hba->shutting_down ||
6384                 !hba->ufs_device_wlun ||
6385                 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6386                 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6387                    ufshcd_is_link_broken(hba))));
6388 }
6389
6390 #ifdef CONFIG_PM
6391 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6392 {
6393         struct Scsi_Host *shost = hba->host;
6394         struct scsi_device *sdev;
6395         struct request_queue *q;
6396         int ret;
6397
6398         hba->is_sys_suspended = false;
6399         /*
6400          * Set RPM status of wlun device to RPM_ACTIVE,
6401          * this also clears its runtime error.
6402          */
6403         ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6404
6405         /* hba device might have a runtime error otherwise */
6406         if (ret)
6407                 ret = pm_runtime_set_active(hba->dev);
6408         /*
6409          * If wlun device had runtime error, we also need to resume those
6410          * consumer scsi devices in case any of them has failed to be
6411          * resumed due to supplier runtime resume failure. This is to unblock
6412          * blk_queue_enter in case there are bios waiting inside it.
6413          */
6414         if (!ret) {
6415                 shost_for_each_device(sdev, shost) {
6416                         q = sdev->request_queue;
6417                         if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6418                                        q->rpm_status == RPM_SUSPENDING))
6419                                 pm_request_resume(q->dev);
6420                 }
6421         }
6422 }
6423 #else
6424 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6425 {
6426 }
6427 #endif
6428
6429 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6430 {
6431         struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6432         u32 mode;
6433
6434         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6435
6436         if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6437                 return true;
6438
6439         if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6440                 return true;
6441
6442         return false;
6443 }
6444
6445 static bool ufshcd_abort_one(struct request *rq, void *priv)
6446 {
6447         int *ret = priv;
6448         u32 tag = rq->tag;
6449         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
6450         struct scsi_device *sdev = cmd->device;
6451         struct Scsi_Host *shost = sdev->host;
6452         struct ufs_hba *hba = shost_priv(shost);
6453         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6454         struct ufs_hw_queue *hwq;
6455         unsigned long flags;
6456
6457         *ret = ufshcd_try_to_abort_task(hba, tag);
6458         dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6459                 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6460                 *ret ? "failed" : "succeeded");
6461
6462         /* Release cmd in MCQ mode if abort succeeds */
6463         if (is_mcq_enabled(hba) && (*ret == 0)) {
6464                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
6465                 spin_lock_irqsave(&hwq->cq_lock, flags);
6466                 if (ufshcd_cmd_inflight(lrbp->cmd))
6467                         ufshcd_release_scsi_cmd(hba, lrbp);
6468                 spin_unlock_irqrestore(&hwq->cq_lock, flags);
6469         }
6470
6471         return *ret == 0;
6472 }
6473
6474 /**
6475  * ufshcd_abort_all - Abort all pending commands.
6476  * @hba: Host bus adapter pointer.
6477  *
6478  * Return: true if and only if the host controller needs to be reset.
6479  */
6480 static bool ufshcd_abort_all(struct ufs_hba *hba)
6481 {
6482         int tag, ret = 0;
6483
6484         blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret);
6485         if (ret)
6486                 goto out;
6487
6488         /* Clear pending task management requests */
6489         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6490                 ret = ufshcd_clear_tm_cmd(hba, tag);
6491                 if (ret)
6492                         goto out;
6493         }
6494
6495 out:
6496         /* Complete the requests that are cleared by s/w */
6497         ufshcd_complete_requests(hba, false);
6498
6499         return ret != 0;
6500 }
6501
6502 /**
6503  * ufshcd_err_handler - handle UFS errors that require s/w attention
6504  * @work: pointer to work structure
6505  */
6506 static void ufshcd_err_handler(struct work_struct *work)
6507 {
6508         int retries = MAX_ERR_HANDLER_RETRIES;
6509         struct ufs_hba *hba;
6510         unsigned long flags;
6511         bool needs_restore;
6512         bool needs_reset;
6513         int pmc_err;
6514
6515         hba = container_of(work, struct ufs_hba, eh_work);
6516
6517         dev_info(hba->dev,
6518                  "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6519                  __func__, ufshcd_state_name[hba->ufshcd_state],
6520                  hba->is_powered, hba->shutting_down, hba->saved_err,
6521                  hba->saved_uic_err, hba->force_reset,
6522                  ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6523
6524         down(&hba->host_sem);
6525         spin_lock_irqsave(hba->host->host_lock, flags);
6526         if (ufshcd_err_handling_should_stop(hba)) {
6527                 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6528                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6529                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6530                 up(&hba->host_sem);
6531                 return;
6532         }
6533         ufshcd_set_eh_in_progress(hba);
6534         spin_unlock_irqrestore(hba->host->host_lock, flags);
6535         ufshcd_err_handling_prepare(hba);
6536         /* Complete requests that have door-bell cleared by h/w */
6537         ufshcd_complete_requests(hba, false);
6538         spin_lock_irqsave(hba->host->host_lock, flags);
6539 again:
6540         needs_restore = false;
6541         needs_reset = false;
6542
6543         if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6544                 hba->ufshcd_state = UFSHCD_STATE_RESET;
6545         /*
6546          * A full reset and restore might have happened after preparation
6547          * is finished, double check whether we should stop.
6548          */
6549         if (ufshcd_err_handling_should_stop(hba))
6550                 goto skip_err_handling;
6551
6552         if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6553                 bool ret;
6554
6555                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6556                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6557                 ret = ufshcd_quirk_dl_nac_errors(hba);
6558                 spin_lock_irqsave(hba->host->host_lock, flags);
6559                 if (!ret && ufshcd_err_handling_should_stop(hba))
6560                         goto skip_err_handling;
6561         }
6562
6563         if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6564             (hba->saved_uic_err &&
6565              (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6566                 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6567
6568                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6569                 ufshcd_print_host_state(hba);
6570                 ufshcd_print_pwr_info(hba);
6571                 ufshcd_print_evt_hist(hba);
6572                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6573                 ufshcd_print_trs_all(hba, pr_prdt);
6574                 spin_lock_irqsave(hba->host->host_lock, flags);
6575         }
6576
6577         /*
6578          * if host reset is required then skip clearing the pending
6579          * transfers forcefully because they will get cleared during
6580          * host reset and restore
6581          */
6582         if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6583             ufshcd_is_saved_err_fatal(hba) ||
6584             ((hba->saved_err & UIC_ERROR) &&
6585              (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6586                                     UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6587                 needs_reset = true;
6588                 goto do_reset;
6589         }
6590
6591         /*
6592          * If LINERESET was caught, UFS might have been put to PWM mode,
6593          * check if power mode restore is needed.
6594          */
6595         if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6596                 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6597                 if (!hba->saved_uic_err)
6598                         hba->saved_err &= ~UIC_ERROR;
6599                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6600                 if (ufshcd_is_pwr_mode_restore_needed(hba))
6601                         needs_restore = true;
6602                 spin_lock_irqsave(hba->host->host_lock, flags);
6603                 if (!hba->saved_err && !needs_restore)
6604                         goto skip_err_handling;
6605         }
6606
6607         hba->silence_err_logs = true;
6608         /* release lock as clear command might sleep */
6609         spin_unlock_irqrestore(hba->host->host_lock, flags);
6610
6611         needs_reset = ufshcd_abort_all(hba);
6612
6613         spin_lock_irqsave(hba->host->host_lock, flags);
6614         hba->silence_err_logs = false;
6615         if (needs_reset)
6616                 goto do_reset;
6617
6618         /*
6619          * After all reqs and tasks are cleared from doorbell,
6620          * now it is safe to retore power mode.
6621          */
6622         if (needs_restore) {
6623                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6624                 /*
6625                  * Hold the scaling lock just in case dev cmds
6626                  * are sent via bsg and/or sysfs.
6627                  */
6628                 down_write(&hba->clk_scaling_lock);
6629                 hba->force_pmc = true;
6630                 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6631                 if (pmc_err) {
6632                         needs_reset = true;
6633                         dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6634                                         __func__, pmc_err);
6635                 }
6636                 hba->force_pmc = false;
6637                 ufshcd_print_pwr_info(hba);
6638                 up_write(&hba->clk_scaling_lock);
6639                 spin_lock_irqsave(hba->host->host_lock, flags);
6640         }
6641
6642 do_reset:
6643         /* Fatal errors need reset */
6644         if (needs_reset) {
6645                 int err;
6646
6647                 hba->force_reset = false;
6648                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6649                 err = ufshcd_reset_and_restore(hba);
6650                 if (err)
6651                         dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6652                                         __func__, err);
6653                 else
6654                         ufshcd_recover_pm_error(hba);
6655                 spin_lock_irqsave(hba->host->host_lock, flags);
6656         }
6657
6658 skip_err_handling:
6659         if (!needs_reset) {
6660                 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6661                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6662                 if (hba->saved_err || hba->saved_uic_err)
6663                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6664                             __func__, hba->saved_err, hba->saved_uic_err);
6665         }
6666         /* Exit in an operational state or dead */
6667         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6668             hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6669                 if (--retries)
6670                         goto again;
6671                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6672         }
6673         ufshcd_clear_eh_in_progress(hba);
6674         spin_unlock_irqrestore(hba->host->host_lock, flags);
6675         ufshcd_err_handling_unprepare(hba);
6676         up(&hba->host_sem);
6677
6678         dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6679                  ufshcd_state_name[hba->ufshcd_state]);
6680 }
6681
6682 /**
6683  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6684  * @hba: per-adapter instance
6685  *
6686  * Return:
6687  *  IRQ_HANDLED - If interrupt is valid
6688  *  IRQ_NONE    - If invalid interrupt
6689  */
6690 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6691 {
6692         u32 reg;
6693         irqreturn_t retval = IRQ_NONE;
6694
6695         /* PHY layer error */
6696         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6697         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6698             (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6699                 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6700                 /*
6701                  * To know whether this error is fatal or not, DB timeout
6702                  * must be checked but this error is handled separately.
6703                  */
6704                 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6705                         dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6706                                         __func__);
6707
6708                 /* Got a LINERESET indication. */
6709                 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6710                         struct uic_command *cmd = NULL;
6711
6712                         hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6713                         if (hba->uic_async_done && hba->active_uic_cmd)
6714                                 cmd = hba->active_uic_cmd;
6715                         /*
6716                          * Ignore the LINERESET during power mode change
6717                          * operation via DME_SET command.
6718                          */
6719                         if (cmd && (cmd->command == UIC_CMD_DME_SET))
6720                                 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6721                 }
6722                 retval |= IRQ_HANDLED;
6723         }
6724
6725         /* PA_INIT_ERROR is fatal and needs UIC reset */
6726         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6727         if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6728             (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6729                 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6730
6731                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6732                         hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6733                 else if (hba->dev_quirks &
6734                                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6735                         if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6736                                 hba->uic_error |=
6737                                         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6738                         else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6739                                 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6740                 }
6741                 retval |= IRQ_HANDLED;
6742         }
6743
6744         /* UIC NL/TL/DME errors needs software retry */
6745         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6746         if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6747             (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6748                 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6749                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6750                 retval |= IRQ_HANDLED;
6751         }
6752
6753         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6754         if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6755             (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6756                 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6757                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6758                 retval |= IRQ_HANDLED;
6759         }
6760
6761         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6762         if ((reg & UIC_DME_ERROR) &&
6763             (reg & UIC_DME_ERROR_CODE_MASK)) {
6764                 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6765                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6766                 retval |= IRQ_HANDLED;
6767         }
6768
6769         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6770                         __func__, hba->uic_error);
6771         return retval;
6772 }
6773
6774 /**
6775  * ufshcd_check_errors - Check for errors that need s/w attention
6776  * @hba: per-adapter instance
6777  * @intr_status: interrupt status generated by the controller
6778  *
6779  * Return:
6780  *  IRQ_HANDLED - If interrupt is valid
6781  *  IRQ_NONE    - If invalid interrupt
6782  */
6783 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6784 {
6785         bool queue_eh_work = false;
6786         irqreturn_t retval = IRQ_NONE;
6787
6788         spin_lock(hba->host->host_lock);
6789         hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6790
6791         if (hba->errors & INT_FATAL_ERRORS) {
6792                 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6793                                        hba->errors);
6794                 queue_eh_work = true;
6795         }
6796
6797         if (hba->errors & UIC_ERROR) {
6798                 hba->uic_error = 0;
6799                 retval = ufshcd_update_uic_error(hba);
6800                 if (hba->uic_error)
6801                         queue_eh_work = true;
6802         }
6803
6804         if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6805                 dev_err(hba->dev,
6806                         "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6807                         __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6808                         "Enter" : "Exit",
6809                         hba->errors, ufshcd_get_upmcrs(hba));
6810                 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6811                                        hba->errors);
6812                 ufshcd_set_link_broken(hba);
6813                 queue_eh_work = true;
6814         }
6815
6816         if (queue_eh_work) {
6817                 /*
6818                  * update the transfer error masks to sticky bits, let's do this
6819                  * irrespective of current ufshcd_state.
6820                  */
6821                 hba->saved_err |= hba->errors;
6822                 hba->saved_uic_err |= hba->uic_error;
6823
6824                 /* dump controller state before resetting */
6825                 if ((hba->saved_err &
6826                      (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6827                     (hba->saved_uic_err &&
6828                      (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6829                         dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6830                                         __func__, hba->saved_err,
6831                                         hba->saved_uic_err);
6832                         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6833                                          "host_regs: ");
6834                         ufshcd_print_pwr_info(hba);
6835                 }
6836                 ufshcd_schedule_eh_work(hba);
6837                 retval |= IRQ_HANDLED;
6838         }
6839         /*
6840          * if (!queue_eh_work) -
6841          * Other errors are either non-fatal where host recovers
6842          * itself without s/w intervention or errors that will be
6843          * handled by the SCSI core layer.
6844          */
6845         hba->errors = 0;
6846         hba->uic_error = 0;
6847         spin_unlock(hba->host->host_lock);
6848         return retval;
6849 }
6850
6851 /**
6852  * ufshcd_tmc_handler - handle task management function completion
6853  * @hba: per adapter instance
6854  *
6855  * Return:
6856  *  IRQ_HANDLED - If interrupt is valid
6857  *  IRQ_NONE    - If invalid interrupt
6858  */
6859 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6860 {
6861         unsigned long flags, pending, issued;
6862         irqreturn_t ret = IRQ_NONE;
6863         int tag;
6864
6865         spin_lock_irqsave(hba->host->host_lock, flags);
6866         pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6867         issued = hba->outstanding_tasks & ~pending;
6868         for_each_set_bit(tag, &issued, hba->nutmrs) {
6869                 struct request *req = hba->tmf_rqs[tag];
6870                 struct completion *c = req->end_io_data;
6871
6872                 complete(c);
6873                 ret = IRQ_HANDLED;
6874         }
6875         spin_unlock_irqrestore(hba->host->host_lock, flags);
6876
6877         return ret;
6878 }
6879
6880 /**
6881  * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
6882  * @hba: per adapter instance
6883  *
6884  * Return: IRQ_HANDLED if interrupt is handled.
6885  */
6886 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
6887 {
6888         struct ufs_hw_queue *hwq;
6889         unsigned long outstanding_cqs;
6890         unsigned int nr_queues;
6891         int i, ret;
6892         u32 events;
6893
6894         ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs);
6895         if (ret)
6896                 outstanding_cqs = (1U << hba->nr_hw_queues) - 1;
6897
6898         /* Exclude the poll queues */
6899         nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
6900         for_each_set_bit(i, &outstanding_cqs, nr_queues) {
6901                 hwq = &hba->uhq[i];
6902
6903                 events = ufshcd_mcq_read_cqis(hba, i);
6904                 if (events)
6905                         ufshcd_mcq_write_cqis(hba, events, i);
6906
6907                 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
6908                         ufshcd_mcq_poll_cqe_lock(hba, hwq);
6909         }
6910
6911         return IRQ_HANDLED;
6912 }
6913
6914 /**
6915  * ufshcd_sl_intr - Interrupt service routine
6916  * @hba: per adapter instance
6917  * @intr_status: contains interrupts generated by the controller
6918  *
6919  * Return:
6920  *  IRQ_HANDLED - If interrupt is valid
6921  *  IRQ_NONE    - If invalid interrupt
6922  */
6923 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6924 {
6925         irqreturn_t retval = IRQ_NONE;
6926
6927         if (intr_status & UFSHCD_UIC_MASK)
6928                 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6929
6930         if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6931                 retval |= ufshcd_check_errors(hba, intr_status);
6932
6933         if (intr_status & UTP_TASK_REQ_COMPL)
6934                 retval |= ufshcd_tmc_handler(hba);
6935
6936         if (intr_status & UTP_TRANSFER_REQ_COMPL)
6937                 retval |= ufshcd_transfer_req_compl(hba);
6938
6939         if (intr_status & MCQ_CQ_EVENT_STATUS)
6940                 retval |= ufshcd_handle_mcq_cq_events(hba);
6941
6942         return retval;
6943 }
6944
6945 /**
6946  * ufshcd_intr - Main interrupt service routine
6947  * @irq: irq number
6948  * @__hba: pointer to adapter instance
6949  *
6950  * Return:
6951  *  IRQ_HANDLED - If interrupt is valid
6952  *  IRQ_NONE    - If invalid interrupt
6953  */
6954 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6955 {
6956         u32 intr_status, enabled_intr_status = 0;
6957         irqreturn_t retval = IRQ_NONE;
6958         struct ufs_hba *hba = __hba;
6959         int retries = hba->nutrs;
6960
6961         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6962         hba->ufs_stats.last_intr_status = intr_status;
6963         hba->ufs_stats.last_intr_ts = local_clock();
6964
6965         /*
6966          * There could be max of hba->nutrs reqs in flight and in worst case
6967          * if the reqs get finished 1 by 1 after the interrupt status is
6968          * read, make sure we handle them by checking the interrupt status
6969          * again in a loop until we process all of the reqs before returning.
6970          */
6971         while (intr_status && retries--) {
6972                 enabled_intr_status =
6973                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6974                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6975                 if (enabled_intr_status)
6976                         retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6977
6978                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6979         }
6980
6981         if (enabled_intr_status && retval == IRQ_NONE &&
6982             (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6983              hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6984                 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6985                                         __func__,
6986                                         intr_status,
6987                                         hba->ufs_stats.last_intr_status,
6988                                         enabled_intr_status);
6989                 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6990         }
6991
6992         return retval;
6993 }
6994
6995 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6996 {
6997         int err = 0;
6998         u32 mask = 1 << tag;
6999         unsigned long flags;
7000
7001         if (!test_bit(tag, &hba->outstanding_tasks))
7002                 goto out;
7003
7004         spin_lock_irqsave(hba->host->host_lock, flags);
7005         ufshcd_utmrl_clear(hba, tag);
7006         spin_unlock_irqrestore(hba->host->host_lock, flags);
7007
7008         /* poll for max. 1 sec to clear door bell register by h/w */
7009         err = ufshcd_wait_for_register(hba,
7010                         REG_UTP_TASK_REQ_DOOR_BELL,
7011                         mask, 0, 1000, 1000);
7012
7013         dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
7014                 tag, err < 0 ? "failed" : "succeeded");
7015
7016 out:
7017         return err;
7018 }
7019
7020 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
7021                 struct utp_task_req_desc *treq, u8 tm_function)
7022 {
7023         struct request_queue *q = hba->tmf_queue;
7024         struct Scsi_Host *host = hba->host;
7025         DECLARE_COMPLETION_ONSTACK(wait);
7026         struct request *req;
7027         unsigned long flags;
7028         int task_tag, err;
7029
7030         /*
7031          * blk_mq_alloc_request() is used here only to get a free tag.
7032          */
7033         req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
7034         if (IS_ERR(req))
7035                 return PTR_ERR(req);
7036
7037         req->end_io_data = &wait;
7038         ufshcd_hold(hba);
7039
7040         spin_lock_irqsave(host->host_lock, flags);
7041
7042         task_tag = req->tag;
7043         hba->tmf_rqs[req->tag] = req;
7044         treq->upiu_req.req_header.task_tag = task_tag;
7045
7046         memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
7047         ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
7048
7049         /* send command to the controller */
7050         __set_bit(task_tag, &hba->outstanding_tasks);
7051
7052         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
7053         /* Make sure that doorbell is committed immediately */
7054         wmb();
7055
7056         spin_unlock_irqrestore(host->host_lock, flags);
7057
7058         ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
7059
7060         /* wait until the task management command is completed */
7061         err = wait_for_completion_io_timeout(&wait,
7062                         msecs_to_jiffies(TM_CMD_TIMEOUT));
7063         if (!err) {
7064                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
7065                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
7066                                 __func__, tm_function);
7067                 if (ufshcd_clear_tm_cmd(hba, task_tag))
7068                         dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
7069                                         __func__, task_tag);
7070                 err = -ETIMEDOUT;
7071         } else {
7072                 err = 0;
7073                 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
7074
7075                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7076         }
7077
7078         spin_lock_irqsave(hba->host->host_lock, flags);
7079         hba->tmf_rqs[req->tag] = NULL;
7080         __clear_bit(task_tag, &hba->outstanding_tasks);
7081         spin_unlock_irqrestore(hba->host->host_lock, flags);
7082
7083         ufshcd_release(hba);
7084         blk_mq_free_request(req);
7085
7086         return err;
7087 }
7088
7089 /**
7090  * ufshcd_issue_tm_cmd - issues task management commands to controller
7091  * @hba: per adapter instance
7092  * @lun_id: LUN ID to which TM command is sent
7093  * @task_id: task ID to which the TM command is applicable
7094  * @tm_function: task management function opcode
7095  * @tm_response: task management service response return value
7096  *
7097  * Return: non-zero value on error, zero on success.
7098  */
7099 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
7100                 u8 tm_function, u8 *tm_response)
7101 {
7102         struct utp_task_req_desc treq = { };
7103         enum utp_ocs ocs_value;
7104         int err;
7105
7106         /* Configure task request descriptor */
7107         treq.header.interrupt = 1;
7108         treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7109
7110         /* Configure task request UPIU */
7111         treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ;
7112         treq.upiu_req.req_header.lun = lun_id;
7113         treq.upiu_req.req_header.tm_function = tm_function;
7114
7115         /*
7116          * The host shall provide the same value for LUN field in the basic
7117          * header and for Input Parameter.
7118          */
7119         treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
7120         treq.upiu_req.input_param2 = cpu_to_be32(task_id);
7121
7122         err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
7123         if (err == -ETIMEDOUT)
7124                 return err;
7125
7126         ocs_value = treq.header.ocs & MASK_OCS;
7127         if (ocs_value != OCS_SUCCESS)
7128                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
7129                                 __func__, ocs_value);
7130         else if (tm_response)
7131                 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
7132                                 MASK_TM_SERVICE_RESP;
7133         return err;
7134 }
7135
7136 /**
7137  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
7138  * @hba:        per-adapter instance
7139  * @req_upiu:   upiu request
7140  * @rsp_upiu:   upiu reply
7141  * @desc_buff:  pointer to descriptor buffer, NULL if NA
7142  * @buff_len:   descriptor size, 0 if NA
7143  * @cmd_type:   specifies the type (NOP, Query...)
7144  * @desc_op:    descriptor operation
7145  *
7146  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
7147  * Therefore, it "rides" the device management infrastructure: uses its tag and
7148  * tasks work queues.
7149  *
7150  * Since there is only one available tag for device management commands,
7151  * the caller is expected to hold the hba->dev_cmd.lock mutex.
7152  *
7153  * Return: 0 upon success; < 0 upon failure.
7154  */
7155 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
7156                                         struct utp_upiu_req *req_upiu,
7157                                         struct utp_upiu_req *rsp_upiu,
7158                                         u8 *desc_buff, int *buff_len,
7159                                         enum dev_cmd_type cmd_type,
7160                                         enum query_opcode desc_op)
7161 {
7162         DECLARE_COMPLETION_ONSTACK(wait);
7163         const u32 tag = hba->reserved_slot;
7164         struct ufshcd_lrb *lrbp;
7165         int err = 0;
7166         u8 upiu_flags;
7167
7168         /* Protects use of hba->reserved_slot. */
7169         lockdep_assert_held(&hba->dev_cmd.lock);
7170
7171         down_read(&hba->clk_scaling_lock);
7172
7173         lrbp = &hba->lrb[tag];
7174         lrbp->cmd = NULL;
7175         lrbp->task_tag = tag;
7176         lrbp->lun = 0;
7177         lrbp->intr_cmd = true;
7178         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
7179         hba->dev_cmd.type = cmd_type;
7180
7181         if (hba->ufs_version <= ufshci_version(1, 1))
7182                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
7183         else
7184                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
7185
7186         /* update the task tag in the request upiu */
7187         req_upiu->header.task_tag = tag;
7188
7189         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0);
7190
7191         /* just copy the upiu request as it is */
7192         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7193         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
7194                 /* The Data Segment Area is optional depending upon the query
7195                  * function value. for WRITE DESCRIPTOR, the data segment
7196                  * follows right after the tsf.
7197                  */
7198                 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
7199                 *buff_len = 0;
7200         }
7201
7202         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7203
7204         hba->dev_cmd.complete = &wait;
7205
7206         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
7207
7208         ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
7209         /*
7210          * ignore the returning value here - ufshcd_check_query_response is
7211          * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
7212          * read the response directly ignoring all errors.
7213          */
7214         ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
7215
7216         /* just copy the upiu response as it is */
7217         memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7218         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
7219                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
7220                 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
7221                                            .data_segment_length);
7222
7223                 if (*buff_len >= resp_len) {
7224                         memcpy(desc_buff, descp, resp_len);
7225                         *buff_len = resp_len;
7226                 } else {
7227                         dev_warn(hba->dev,
7228                                  "%s: rsp size %d is bigger than buffer size %d",
7229                                  __func__, resp_len, *buff_len);
7230                         *buff_len = 0;
7231                         err = -EINVAL;
7232                 }
7233         }
7234         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
7235                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
7236
7237         up_read(&hba->clk_scaling_lock);
7238         return err;
7239 }
7240
7241 /**
7242  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
7243  * @hba:        per-adapter instance
7244  * @req_upiu:   upiu request
7245  * @rsp_upiu:   upiu reply - only 8 DW as we do not support scsi commands
7246  * @msgcode:    message code, one of UPIU Transaction Codes Initiator to Target
7247  * @desc_buff:  pointer to descriptor buffer, NULL if NA
7248  * @buff_len:   descriptor size, 0 if NA
7249  * @desc_op:    descriptor operation
7250  *
7251  * Supports UTP Transfer requests (nop and query), and UTP Task
7252  * Management requests.
7253  * It is up to the caller to fill the upiu conent properly, as it will
7254  * be copied without any further input validations.
7255  *
7256  * Return: 0 upon success; < 0 upon failure.
7257  */
7258 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
7259                              struct utp_upiu_req *req_upiu,
7260                              struct utp_upiu_req *rsp_upiu,
7261                              enum upiu_request_transaction msgcode,
7262                              u8 *desc_buff, int *buff_len,
7263                              enum query_opcode desc_op)
7264 {
7265         int err;
7266         enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
7267         struct utp_task_req_desc treq = { };
7268         enum utp_ocs ocs_value;
7269         u8 tm_f = req_upiu->header.tm_function;
7270
7271         switch (msgcode) {
7272         case UPIU_TRANSACTION_NOP_OUT:
7273                 cmd_type = DEV_CMD_TYPE_NOP;
7274                 fallthrough;
7275         case UPIU_TRANSACTION_QUERY_REQ:
7276                 ufshcd_hold(hba);
7277                 mutex_lock(&hba->dev_cmd.lock);
7278                 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
7279                                                    desc_buff, buff_len,
7280                                                    cmd_type, desc_op);
7281                 mutex_unlock(&hba->dev_cmd.lock);
7282                 ufshcd_release(hba);
7283
7284                 break;
7285         case UPIU_TRANSACTION_TASK_REQ:
7286                 treq.header.interrupt = 1;
7287                 treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7288
7289                 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
7290
7291                 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
7292                 if (err == -ETIMEDOUT)
7293                         break;
7294
7295                 ocs_value = treq.header.ocs & MASK_OCS;
7296                 if (ocs_value != OCS_SUCCESS) {
7297                         dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7298                                 ocs_value);
7299                         break;
7300                 }
7301
7302                 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
7303
7304                 break;
7305         default:
7306                 err = -EINVAL;
7307
7308                 break;
7309         }
7310
7311         return err;
7312 }
7313
7314 /**
7315  * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request
7316  * @hba:        per adapter instance
7317  * @req_upiu:   upiu request
7318  * @rsp_upiu:   upiu reply
7319  * @req_ehs:    EHS field which contains Advanced RPMB Request Message
7320  * @rsp_ehs:    EHS field which returns Advanced RPMB Response Message
7321  * @sg_cnt:     The number of sg lists actually used
7322  * @sg_list:    Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation
7323  * @dir:        DMA direction
7324  *
7325  * Return: zero on success, non-zero on failure.
7326  */
7327 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu,
7328                          struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs,
7329                          struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list,
7330                          enum dma_data_direction dir)
7331 {
7332         DECLARE_COMPLETION_ONSTACK(wait);
7333         const u32 tag = hba->reserved_slot;
7334         struct ufshcd_lrb *lrbp;
7335         int err = 0;
7336         int result;
7337         u8 upiu_flags;
7338         u8 *ehs_data;
7339         u16 ehs_len;
7340
7341         /* Protects use of hba->reserved_slot. */
7342         ufshcd_hold(hba);
7343         mutex_lock(&hba->dev_cmd.lock);
7344         down_read(&hba->clk_scaling_lock);
7345
7346         lrbp = &hba->lrb[tag];
7347         lrbp->cmd = NULL;
7348         lrbp->task_tag = tag;
7349         lrbp->lun = UFS_UPIU_RPMB_WLUN;
7350
7351         lrbp->intr_cmd = true;
7352         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
7353         hba->dev_cmd.type = DEV_CMD_TYPE_RPMB;
7354
7355         /* Advanced RPMB starts from UFS 4.0, so its command type is UTP_CMD_TYPE_UFS_STORAGE */
7356         lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
7357
7358         /*
7359          * According to UFSHCI 4.0 specification page 24, if EHSLUTRDS is 0, host controller takes
7360          * EHS length from CMD UPIU, and SW driver use EHS Length field in CMD UPIU. if it is 1,
7361          * HW controller takes EHS length from UTRD.
7362          */
7363         if (hba->capabilities & MASK_EHSLUTRD_SUPPORTED)
7364                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, dir, 2);
7365         else
7366                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, dir, 0);
7367
7368         /* update the task tag */
7369         req_upiu->header.task_tag = tag;
7370
7371         /* copy the UPIU(contains CDB) request as it is */
7372         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7373         /* Copy EHS, starting with byte32, immediately after the CDB package */
7374         memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs));
7375
7376         if (dir != DMA_NONE && sg_list)
7377                 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list);
7378
7379         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7380
7381         hba->dev_cmd.complete = &wait;
7382
7383         ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
7384
7385         err = ufshcd_wait_for_dev_cmd(hba, lrbp, ADVANCED_RPMB_REQ_TIMEOUT);
7386
7387         if (!err) {
7388                 /* Just copy the upiu response as it is */
7389                 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7390                 /* Get the response UPIU result */
7391                 result = (lrbp->ucd_rsp_ptr->header.response << 8) |
7392                         lrbp->ucd_rsp_ptr->header.status;
7393
7394                 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length;
7395                 /*
7396                  * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data
7397                  * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB
7398                  * Message is 02h
7399                  */
7400                 if (ehs_len == 2 && rsp_ehs) {
7401                         /*
7402                          * ucd_rsp_ptr points to a buffer with a length of 512 bytes
7403                          * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32
7404                          */
7405                         ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE;
7406                         memcpy(rsp_ehs, ehs_data, ehs_len * 32);
7407                 }
7408         }
7409
7410         up_read(&hba->clk_scaling_lock);
7411         mutex_unlock(&hba->dev_cmd.lock);
7412         ufshcd_release(hba);
7413         return err ? : result;
7414 }
7415
7416 /**
7417  * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7418  * @cmd: SCSI command pointer
7419  *
7420  * Return: SUCCESS or FAILED.
7421  */
7422 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7423 {
7424         unsigned long flags, pending_reqs = 0, not_cleared = 0;
7425         struct Scsi_Host *host;
7426         struct ufs_hba *hba;
7427         struct ufs_hw_queue *hwq;
7428         struct ufshcd_lrb *lrbp;
7429         u32 pos, not_cleared_mask = 0;
7430         int err;
7431         u8 resp = 0xF, lun;
7432
7433         host = cmd->device->host;
7434         hba = shost_priv(host);
7435
7436         lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7437         err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7438         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7439                 if (!err)
7440                         err = resp;
7441                 goto out;
7442         }
7443
7444         if (is_mcq_enabled(hba)) {
7445                 for (pos = 0; pos < hba->nutrs; pos++) {
7446                         lrbp = &hba->lrb[pos];
7447                         if (ufshcd_cmd_inflight(lrbp->cmd) &&
7448                             lrbp->lun == lun) {
7449                                 ufshcd_clear_cmd(hba, pos);
7450                                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
7451                                 ufshcd_mcq_poll_cqe_lock(hba, hwq);
7452                         }
7453                 }
7454                 err = 0;
7455                 goto out;
7456         }
7457
7458         /* clear the commands that were pending for corresponding LUN */
7459         spin_lock_irqsave(&hba->outstanding_lock, flags);
7460         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7461                 if (hba->lrb[pos].lun == lun)
7462                         __set_bit(pos, &pending_reqs);
7463         hba->outstanding_reqs &= ~pending_reqs;
7464         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7465
7466         for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
7467                 if (ufshcd_clear_cmd(hba, pos) < 0) {
7468                         spin_lock_irqsave(&hba->outstanding_lock, flags);
7469                         not_cleared = 1U << pos &
7470                                 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7471                         hba->outstanding_reqs |= not_cleared;
7472                         not_cleared_mask |= not_cleared;
7473                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7474
7475                         dev_err(hba->dev, "%s: failed to clear request %d\n",
7476                                 __func__, pos);
7477                 }
7478         }
7479         __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
7480
7481 out:
7482         hba->req_abort_count = 0;
7483         ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7484         if (!err) {
7485                 err = SUCCESS;
7486         } else {
7487                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7488                 err = FAILED;
7489         }
7490         return err;
7491 }
7492
7493 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7494 {
7495         struct ufshcd_lrb *lrbp;
7496         int tag;
7497
7498         for_each_set_bit(tag, &bitmap, hba->nutrs) {
7499                 lrbp = &hba->lrb[tag];
7500                 lrbp->req_abort_skip = true;
7501         }
7502 }
7503
7504 /**
7505  * ufshcd_try_to_abort_task - abort a specific task
7506  * @hba: Pointer to adapter instance
7507  * @tag: Task tag/index to be aborted
7508  *
7509  * Abort the pending command in device by sending UFS_ABORT_TASK task management
7510  * command, and in host controller by clearing the door-bell register. There can
7511  * be race between controller sending the command to the device while abort is
7512  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7513  * really issued and then try to abort it.
7514  *
7515  * Return: zero on success, non-zero on failure.
7516  */
7517 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7518 {
7519         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7520         int err = 0;
7521         int poll_cnt;
7522         u8 resp = 0xF;
7523         u32 reg;
7524
7525         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7526                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7527                                 UFS_QUERY_TASK, &resp);
7528                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7529                         /* cmd pending in the device */
7530                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7531                                 __func__, tag);
7532                         break;
7533                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7534                         /*
7535                          * cmd not pending in the device, check if it is
7536                          * in transition.
7537                          */
7538                         dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7539                                 __func__, tag);
7540                         if (is_mcq_enabled(hba)) {
7541                                 /* MCQ mode */
7542                                 if (ufshcd_cmd_inflight(lrbp->cmd)) {
7543                                         /* sleep for max. 200us same delay as in SDB mode */
7544                                         usleep_range(100, 200);
7545                                         continue;
7546                                 }
7547                                 /* command completed already */
7548                                 dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
7549                                         __func__, tag);
7550                                 goto out;
7551                         }
7552
7553                         /* Single Doorbell Mode */
7554                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7555                         if (reg & (1 << tag)) {
7556                                 /* sleep for max. 200us to stabilize */
7557                                 usleep_range(100, 200);
7558                                 continue;
7559                         }
7560                         /* command completed already */
7561                         dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7562                                 __func__, tag);
7563                         goto out;
7564                 } else {
7565                         dev_err(hba->dev,
7566                                 "%s: no response from device. tag = %d, err %d\n",
7567                                 __func__, tag, err);
7568                         if (!err)
7569                                 err = resp; /* service response error */
7570                         goto out;
7571                 }
7572         }
7573
7574         if (!poll_cnt) {
7575                 err = -EBUSY;
7576                 goto out;
7577         }
7578
7579         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7580                         UFS_ABORT_TASK, &resp);
7581         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7582                 if (!err) {
7583                         err = resp; /* service response error */
7584                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7585                                 __func__, tag, err);
7586                 }
7587                 goto out;
7588         }
7589
7590         err = ufshcd_clear_cmd(hba, tag);
7591         if (err)
7592                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7593                         __func__, tag, err);
7594
7595 out:
7596         return err;
7597 }
7598
7599 /**
7600  * ufshcd_abort - scsi host template eh_abort_handler callback
7601  * @cmd: SCSI command pointer
7602  *
7603  * Return: SUCCESS or FAILED.
7604  */
7605 static int ufshcd_abort(struct scsi_cmnd *cmd)
7606 {
7607         struct Scsi_Host *host = cmd->device->host;
7608         struct ufs_hba *hba = shost_priv(host);
7609         int tag = scsi_cmd_to_rq(cmd)->tag;
7610         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7611         unsigned long flags;
7612         int err = FAILED;
7613         bool outstanding;
7614         u32 reg;
7615
7616         ufshcd_hold(hba);
7617
7618         if (!is_mcq_enabled(hba)) {
7619                 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7620                 if (!test_bit(tag, &hba->outstanding_reqs)) {
7621                         /* If command is already aborted/completed, return FAILED. */
7622                         dev_err(hba->dev,
7623                                 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7624                                 __func__, tag, hba->outstanding_reqs, reg);
7625                         goto release;
7626                 }
7627         }
7628
7629         /* Print Transfer Request of aborted task */
7630         dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7631
7632         /*
7633          * Print detailed info about aborted request.
7634          * As more than one request might get aborted at the same time,
7635          * print full information only for the first aborted request in order
7636          * to reduce repeated printouts. For other aborted requests only print
7637          * basic details.
7638          */
7639         scsi_print_command(cmd);
7640         if (!hba->req_abort_count) {
7641                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7642                 ufshcd_print_evt_hist(hba);
7643                 ufshcd_print_host_state(hba);
7644                 ufshcd_print_pwr_info(hba);
7645                 ufshcd_print_tr(hba, tag, true);
7646         } else {
7647                 ufshcd_print_tr(hba, tag, false);
7648         }
7649         hba->req_abort_count++;
7650
7651         if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) {
7652                 /* only execute this code in single doorbell mode */
7653                 dev_err(hba->dev,
7654                 "%s: cmd was completed, but without a notifying intr, tag = %d",
7655                 __func__, tag);
7656                 __ufshcd_transfer_req_compl(hba, 1UL << tag);
7657                 goto release;
7658         }
7659
7660         /*
7661          * Task abort to the device W-LUN is illegal. When this command
7662          * will fail, due to spec violation, scsi err handling next step
7663          * will be to send LU reset which, again, is a spec violation.
7664          * To avoid these unnecessary/illegal steps, first we clean up
7665          * the lrb taken by this cmd and re-set it in outstanding_reqs,
7666          * then queue the eh_work and bail.
7667          */
7668         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7669                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7670
7671                 spin_lock_irqsave(host->host_lock, flags);
7672                 hba->force_reset = true;
7673                 ufshcd_schedule_eh_work(hba);
7674                 spin_unlock_irqrestore(host->host_lock, flags);
7675                 goto release;
7676         }
7677
7678         if (is_mcq_enabled(hba)) {
7679                 /* MCQ mode. Branch off to handle abort for mcq mode */
7680                 err = ufshcd_mcq_abort(cmd);
7681                 goto release;
7682         }
7683
7684         /* Skip task abort in case previous aborts failed and report failure */
7685         if (lrbp->req_abort_skip) {
7686                 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7687                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7688                 goto release;
7689         }
7690
7691         err = ufshcd_try_to_abort_task(hba, tag);
7692         if (err) {
7693                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7694                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7695                 err = FAILED;
7696                 goto release;
7697         }
7698
7699         /*
7700          * Clear the corresponding bit from outstanding_reqs since the command
7701          * has been aborted successfully.
7702          */
7703         spin_lock_irqsave(&hba->outstanding_lock, flags);
7704         outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7705         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7706
7707         if (outstanding)
7708                 ufshcd_release_scsi_cmd(hba, lrbp);
7709
7710         err = SUCCESS;
7711
7712 release:
7713         /* Matches the ufshcd_hold() call at the start of this function. */
7714         ufshcd_release(hba);
7715         return err;
7716 }
7717
7718 /**
7719  * ufshcd_host_reset_and_restore - reset and restore host controller
7720  * @hba: per-adapter instance
7721  *
7722  * Note that host controller reset may issue DME_RESET to
7723  * local and remote (device) Uni-Pro stack and the attributes
7724  * are reset to default state.
7725  *
7726  * Return: zero on success, non-zero on failure.
7727  */
7728 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7729 {
7730         int err;
7731
7732         /*
7733          * Stop the host controller and complete the requests
7734          * cleared by h/w
7735          */
7736         ufshcd_hba_stop(hba);
7737         hba->silence_err_logs = true;
7738         ufshcd_complete_requests(hba, true);
7739         hba->silence_err_logs = false;
7740
7741         /* scale up clocks to max frequency before full reinitialization */
7742         ufshcd_scale_clks(hba, ULONG_MAX, true);
7743
7744         err = ufshcd_hba_enable(hba);
7745
7746         /* Establish the link again and restore the device */
7747         if (!err)
7748                 err = ufshcd_probe_hba(hba, false);
7749
7750         if (err)
7751                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7752         ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7753         return err;
7754 }
7755
7756 /**
7757  * ufshcd_reset_and_restore - reset and re-initialize host/device
7758  * @hba: per-adapter instance
7759  *
7760  * Reset and recover device, host and re-establish link. This
7761  * is helpful to recover the communication in fatal error conditions.
7762  *
7763  * Return: zero on success, non-zero on failure.
7764  */
7765 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7766 {
7767         u32 saved_err = 0;
7768         u32 saved_uic_err = 0;
7769         int err = 0;
7770         unsigned long flags;
7771         int retries = MAX_HOST_RESET_RETRIES;
7772
7773         spin_lock_irqsave(hba->host->host_lock, flags);
7774         do {
7775                 /*
7776                  * This is a fresh start, cache and clear saved error first,
7777                  * in case new error generated during reset and restore.
7778                  */
7779                 saved_err |= hba->saved_err;
7780                 saved_uic_err |= hba->saved_uic_err;
7781                 hba->saved_err = 0;
7782                 hba->saved_uic_err = 0;
7783                 hba->force_reset = false;
7784                 hba->ufshcd_state = UFSHCD_STATE_RESET;
7785                 spin_unlock_irqrestore(hba->host->host_lock, flags);
7786
7787                 /* Reset the attached device */
7788                 ufshcd_device_reset(hba);
7789
7790                 err = ufshcd_host_reset_and_restore(hba);
7791
7792                 spin_lock_irqsave(hba->host->host_lock, flags);
7793                 if (err)
7794                         continue;
7795                 /* Do not exit unless operational or dead */
7796                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7797                     hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7798                     hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7799                         err = -EAGAIN;
7800         } while (err && --retries);
7801
7802         /*
7803          * Inform scsi mid-layer that we did reset and allow to handle
7804          * Unit Attention properly.
7805          */
7806         scsi_report_bus_reset(hba->host, 0);
7807         if (err) {
7808                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7809                 hba->saved_err |= saved_err;
7810                 hba->saved_uic_err |= saved_uic_err;
7811         }
7812         spin_unlock_irqrestore(hba->host->host_lock, flags);
7813
7814         return err;
7815 }
7816
7817 /**
7818  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7819  * @cmd: SCSI command pointer
7820  *
7821  * Return: SUCCESS or FAILED.
7822  */
7823 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7824 {
7825         int err = SUCCESS;
7826         unsigned long flags;
7827         struct ufs_hba *hba;
7828
7829         hba = shost_priv(cmd->device->host);
7830
7831         /*
7832          * If runtime PM sent SSU and got a timeout, scsi_error_handler is
7833          * stuck in this function waiting for flush_work(&hba->eh_work). And
7834          * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do
7835          * ufshcd_link_recovery instead of eh_work to prevent deadlock.
7836          */
7837         if (hba->pm_op_in_progress) {
7838                 if (ufshcd_link_recovery(hba))
7839                         err = FAILED;
7840
7841                 return err;
7842         }
7843
7844         spin_lock_irqsave(hba->host->host_lock, flags);
7845         hba->force_reset = true;
7846         ufshcd_schedule_eh_work(hba);
7847         dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7848         spin_unlock_irqrestore(hba->host->host_lock, flags);
7849
7850         flush_work(&hba->eh_work);
7851
7852         spin_lock_irqsave(hba->host->host_lock, flags);
7853         if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7854                 err = FAILED;
7855         spin_unlock_irqrestore(hba->host->host_lock, flags);
7856
7857         return err;
7858 }
7859
7860 /**
7861  * ufshcd_get_max_icc_level - calculate the ICC level
7862  * @sup_curr_uA: max. current supported by the regulator
7863  * @start_scan: row at the desc table to start scan from
7864  * @buff: power descriptor buffer
7865  *
7866  * Return: calculated max ICC level for specific regulator.
7867  */
7868 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan,
7869                                     const char *buff)
7870 {
7871         int i;
7872         int curr_uA;
7873         u16 data;
7874         u16 unit;
7875
7876         for (i = start_scan; i >= 0; i--) {
7877                 data = get_unaligned_be16(&buff[2 * i]);
7878                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7879                                                 ATTR_ICC_LVL_UNIT_OFFSET;
7880                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7881                 switch (unit) {
7882                 case UFSHCD_NANO_AMP:
7883                         curr_uA = curr_uA / 1000;
7884                         break;
7885                 case UFSHCD_MILI_AMP:
7886                         curr_uA = curr_uA * 1000;
7887                         break;
7888                 case UFSHCD_AMP:
7889                         curr_uA = curr_uA * 1000 * 1000;
7890                         break;
7891                 case UFSHCD_MICRO_AMP:
7892                 default:
7893                         break;
7894                 }
7895                 if (sup_curr_uA >= curr_uA)
7896                         break;
7897         }
7898         if (i < 0) {
7899                 i = 0;
7900                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7901         }
7902
7903         return (u32)i;
7904 }
7905
7906 /**
7907  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7908  * In case regulators are not initialized we'll return 0
7909  * @hba: per-adapter instance
7910  * @desc_buf: power descriptor buffer to extract ICC levels from.
7911  *
7912  * Return: calculated ICC level.
7913  */
7914 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7915                                                 const u8 *desc_buf)
7916 {
7917         u32 icc_level = 0;
7918
7919         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7920                                                 !hba->vreg_info.vccq2) {
7921                 /*
7922                  * Using dev_dbg to avoid messages during runtime PM to avoid
7923                  * never-ending cycles of messages written back to storage by
7924                  * user space causing runtime resume, causing more messages and
7925                  * so on.
7926                  */
7927                 dev_dbg(hba->dev,
7928                         "%s: Regulator capability was not set, actvIccLevel=%d",
7929                                                         __func__, icc_level);
7930                 goto out;
7931         }
7932
7933         if (hba->vreg_info.vcc->max_uA)
7934                 icc_level = ufshcd_get_max_icc_level(
7935                                 hba->vreg_info.vcc->max_uA,
7936                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7937                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7938
7939         if (hba->vreg_info.vccq->max_uA)
7940                 icc_level = ufshcd_get_max_icc_level(
7941                                 hba->vreg_info.vccq->max_uA,
7942                                 icc_level,
7943                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7944
7945         if (hba->vreg_info.vccq2->max_uA)
7946                 icc_level = ufshcd_get_max_icc_level(
7947                                 hba->vreg_info.vccq2->max_uA,
7948                                 icc_level,
7949                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7950 out:
7951         return icc_level;
7952 }
7953
7954 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7955 {
7956         int ret;
7957         u8 *desc_buf;
7958         u32 icc_level;
7959
7960         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7961         if (!desc_buf)
7962                 return;
7963
7964         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7965                                      desc_buf, QUERY_DESC_MAX_SIZE);
7966         if (ret) {
7967                 dev_err(hba->dev,
7968                         "%s: Failed reading power descriptor ret = %d",
7969                         __func__, ret);
7970                 goto out;
7971         }
7972
7973         icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf);
7974         dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7975
7976         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7977                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7978
7979         if (ret)
7980                 dev_err(hba->dev,
7981                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7982                         __func__, icc_level, ret);
7983
7984 out:
7985         kfree(desc_buf);
7986 }
7987
7988 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7989 {
7990         scsi_autopm_get_device(sdev);
7991         blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7992         if (sdev->rpm_autosuspend)
7993                 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7994                                                  RPM_AUTOSUSPEND_DELAY_MS);
7995         scsi_autopm_put_device(sdev);
7996 }
7997
7998 /**
7999  * ufshcd_scsi_add_wlus - Adds required W-LUs
8000  * @hba: per-adapter instance
8001  *
8002  * UFS device specification requires the UFS devices to support 4 well known
8003  * logical units:
8004  *      "REPORT_LUNS" (address: 01h)
8005  *      "UFS Device" (address: 50h)
8006  *      "RPMB" (address: 44h)
8007  *      "BOOT" (address: 30h)
8008  * UFS device's power management needs to be controlled by "POWER CONDITION"
8009  * field of SSU (START STOP UNIT) command. But this "power condition" field
8010  * will take effect only when its sent to "UFS device" well known logical unit
8011  * hence we require the scsi_device instance to represent this logical unit in
8012  * order for the UFS host driver to send the SSU command for power management.
8013  *
8014  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
8015  * Block) LU so user space process can control this LU. User space may also
8016  * want to have access to BOOT LU.
8017  *
8018  * This function adds scsi device instances for each of all well known LUs
8019  * (except "REPORT LUNS" LU).
8020  *
8021  * Return: zero on success (all required W-LUs are added successfully),
8022  * non-zero error value on failure (if failed to add any of the required W-LU).
8023  */
8024 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
8025 {
8026         int ret = 0;
8027         struct scsi_device *sdev_boot, *sdev_rpmb;
8028
8029         hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
8030                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
8031         if (IS_ERR(hba->ufs_device_wlun)) {
8032                 ret = PTR_ERR(hba->ufs_device_wlun);
8033                 hba->ufs_device_wlun = NULL;
8034                 goto out;
8035         }
8036         scsi_device_put(hba->ufs_device_wlun);
8037
8038         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
8039                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
8040         if (IS_ERR(sdev_rpmb)) {
8041                 ret = PTR_ERR(sdev_rpmb);
8042                 goto remove_ufs_device_wlun;
8043         }
8044         ufshcd_blk_pm_runtime_init(sdev_rpmb);
8045         scsi_device_put(sdev_rpmb);
8046
8047         sdev_boot = __scsi_add_device(hba->host, 0, 0,
8048                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
8049         if (IS_ERR(sdev_boot)) {
8050                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
8051         } else {
8052                 ufshcd_blk_pm_runtime_init(sdev_boot);
8053                 scsi_device_put(sdev_boot);
8054         }
8055         goto out;
8056
8057 remove_ufs_device_wlun:
8058         scsi_remove_device(hba->ufs_device_wlun);
8059 out:
8060         return ret;
8061 }
8062
8063 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf)
8064 {
8065         struct ufs_dev_info *dev_info = &hba->dev_info;
8066         u8 lun;
8067         u32 d_lu_wb_buf_alloc;
8068         u32 ext_ufs_feature;
8069
8070         if (!ufshcd_is_wb_allowed(hba))
8071                 return;
8072
8073         /*
8074          * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
8075          * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
8076          * enabled
8077          */
8078         if (!(dev_info->wspecversion >= 0x310 ||
8079               dev_info->wspecversion == 0x220 ||
8080              (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
8081                 goto wb_disabled;
8082
8083         ext_ufs_feature = get_unaligned_be32(desc_buf +
8084                                         DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8085
8086         if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
8087                 goto wb_disabled;
8088
8089         /*
8090          * WB may be supported but not configured while provisioning. The spec
8091          * says, in dedicated wb buffer mode, a max of 1 lun would have wb
8092          * buffer configured.
8093          */
8094         dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
8095
8096         dev_info->b_presrv_uspc_en =
8097                 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
8098
8099         if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
8100                 if (!get_unaligned_be32(desc_buf +
8101                                    DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
8102                         goto wb_disabled;
8103         } else {
8104                 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
8105                         d_lu_wb_buf_alloc = 0;
8106                         ufshcd_read_unit_desc_param(hba,
8107                                         lun,
8108                                         UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
8109                                         (u8 *)&d_lu_wb_buf_alloc,
8110                                         sizeof(d_lu_wb_buf_alloc));
8111                         if (d_lu_wb_buf_alloc) {
8112                                 dev_info->wb_dedicated_lu = lun;
8113                                 break;
8114                         }
8115                 }
8116
8117                 if (!d_lu_wb_buf_alloc)
8118                         goto wb_disabled;
8119         }
8120
8121         if (!ufshcd_is_wb_buf_lifetime_available(hba))
8122                 goto wb_disabled;
8123
8124         return;
8125
8126 wb_disabled:
8127         hba->caps &= ~UFSHCD_CAP_WB_EN;
8128 }
8129
8130 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf)
8131 {
8132         struct ufs_dev_info *dev_info = &hba->dev_info;
8133         u32 ext_ufs_feature;
8134         u8 mask = 0;
8135
8136         if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
8137                 return;
8138
8139         ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8140
8141         if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
8142                 mask |= MASK_EE_TOO_LOW_TEMP;
8143
8144         if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
8145                 mask |= MASK_EE_TOO_HIGH_TEMP;
8146
8147         if (mask) {
8148                 ufshcd_enable_ee(hba, mask);
8149                 ufs_hwmon_probe(hba, mask);
8150         }
8151 }
8152
8153 static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf)
8154 {
8155         struct ufs_dev_info *dev_info = &hba->dev_info;
8156         u32 ext_ufs_feature;
8157         u32 ext_iid_en = 0;
8158         int err;
8159
8160         /* Only UFS-4.0 and above may support EXT_IID */
8161         if (dev_info->wspecversion < 0x400)
8162                 goto out;
8163
8164         ext_ufs_feature = get_unaligned_be32(desc_buf +
8165                                      DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8166         if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP))
8167                 goto out;
8168
8169         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8170                                       QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en);
8171         if (err)
8172                 dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err);
8173
8174 out:
8175         dev_info->b_ext_iid_en = ext_iid_en;
8176 }
8177
8178 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
8179                              const struct ufs_dev_quirk *fixups)
8180 {
8181         const struct ufs_dev_quirk *f;
8182         struct ufs_dev_info *dev_info = &hba->dev_info;
8183
8184         if (!fixups)
8185                 return;
8186
8187         for (f = fixups; f->quirk; f++) {
8188                 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
8189                      f->wmanufacturerid == UFS_ANY_VENDOR) &&
8190                      ((dev_info->model &&
8191                        STR_PRFX_EQUAL(f->model, dev_info->model)) ||
8192                       !strcmp(f->model, UFS_ANY_MODEL)))
8193                         hba->dev_quirks |= f->quirk;
8194         }
8195 }
8196 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
8197
8198 static void ufs_fixup_device_setup(struct ufs_hba *hba)
8199 {
8200         /* fix by general quirk table */
8201         ufshcd_fixup_dev_quirks(hba, ufs_fixups);
8202
8203         /* allow vendors to fix quirks */
8204         ufshcd_vops_fixup_dev_quirks(hba);
8205 }
8206
8207 static void ufshcd_update_rtc(struct ufs_hba *hba)
8208 {
8209         struct timespec64 ts64;
8210         int err;
8211         u32 val;
8212
8213         ktime_get_real_ts64(&ts64);
8214
8215         if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) {
8216                 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__);
8217                 return;
8218         }
8219
8220         /*
8221          * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond
8222          * 2146 is required, it is recommended to choose the relative RTC mode.
8223          */
8224         val = ts64.tv_sec - hba->dev_info.rtc_time_baseline;
8225
8226         ufshcd_rpm_get_sync(hba);
8227         err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED,
8228                                 0, 0, &val);
8229         ufshcd_rpm_put_sync(hba);
8230
8231         if (err)
8232                 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err);
8233         else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
8234                 hba->dev_info.rtc_time_baseline = ts64.tv_sec;
8235 }
8236
8237 static void ufshcd_rtc_work(struct work_struct *work)
8238 {
8239         struct ufs_hba *hba;
8240
8241         hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work);
8242
8243          /* Update RTC only when there are no requests in progress and UFSHCI is operational */
8244         if (!ufshcd_is_ufs_dev_busy(hba) && hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL)
8245                 ufshcd_update_rtc(hba);
8246
8247         if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period)
8248                 schedule_delayed_work(&hba->ufs_rtc_update_work,
8249                                       msecs_to_jiffies(hba->dev_info.rtc_update_period));
8250 }
8251
8252 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf)
8253 {
8254         u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]);
8255         struct ufs_dev_info *dev_info = &hba->dev_info;
8256
8257         if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) {
8258                 dev_info->rtc_type = UFS_RTC_ABSOLUTE;
8259
8260                 /*
8261                  * The concept of measuring time in Linux as the number of seconds elapsed since
8262                  * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st
8263                  * 2010 00:00, here we need to adjust ABS baseline.
8264                  */
8265                 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) -
8266                                                         mktime64(1970, 1, 1, 0, 0, 0);
8267         } else {
8268                 dev_info->rtc_type = UFS_RTC_RELATIVE;
8269                 dev_info->rtc_time_baseline = 0;
8270         }
8271
8272         /*
8273          * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state
8274          * how to calculate the specific update period for each time unit. And we disable periodic
8275          * RTC update work, let user configure by sysfs node according to specific circumstance.
8276          */
8277         dev_info->rtc_update_period = 0;
8278 }
8279
8280 static int ufs_get_device_desc(struct ufs_hba *hba)
8281 {
8282         int err;
8283         u8 model_index;
8284         u8 *desc_buf;
8285         struct ufs_dev_info *dev_info = &hba->dev_info;
8286
8287         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8288         if (!desc_buf) {
8289                 err = -ENOMEM;
8290                 goto out;
8291         }
8292
8293         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
8294                                      QUERY_DESC_MAX_SIZE);
8295         if (err) {
8296                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
8297                         __func__, err);
8298                 goto out;
8299         }
8300
8301         /*
8302          * getting vendor (manufacturerID) and Bank Index in big endian
8303          * format
8304          */
8305         dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
8306                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
8307
8308         /* getting Specification Version in big endian format */
8309         dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
8310                                       desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
8311         dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
8312
8313         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
8314
8315         err = ufshcd_read_string_desc(hba, model_index,
8316                                       &dev_info->model, SD_ASCII_STD);
8317         if (err < 0) {
8318                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
8319                         __func__, err);
8320                 goto out;
8321         }
8322
8323         hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
8324                 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
8325
8326         ufs_fixup_device_setup(hba);
8327
8328         ufshcd_wb_probe(hba, desc_buf);
8329
8330         ufshcd_temp_notif_probe(hba, desc_buf);
8331
8332         ufs_init_rtc(hba, desc_buf);
8333
8334         if (hba->ext_iid_sup)
8335                 ufshcd_ext_iid_probe(hba, desc_buf);
8336
8337         /*
8338          * ufshcd_read_string_desc returns size of the string
8339          * reset the error value
8340          */
8341         err = 0;
8342
8343 out:
8344         kfree(desc_buf);
8345         return err;
8346 }
8347
8348 static void ufs_put_device_desc(struct ufs_hba *hba)
8349 {
8350         struct ufs_dev_info *dev_info = &hba->dev_info;
8351
8352         kfree(dev_info->model);
8353         dev_info->model = NULL;
8354 }
8355
8356 /**
8357  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
8358  * @hba: per-adapter instance
8359  *
8360  * PA_TActivate parameter can be tuned manually if UniPro version is less than
8361  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
8362  * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
8363  * the hibern8 exit latency.
8364  *
8365  * Return: zero on success, non-zero error value on failure.
8366  */
8367 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
8368 {
8369         int ret = 0;
8370         u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
8371
8372         ret = ufshcd_dme_peer_get(hba,
8373                                   UIC_ARG_MIB_SEL(
8374                                         RX_MIN_ACTIVATETIME_CAPABILITY,
8375                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
8376                                   &peer_rx_min_activatetime);
8377         if (ret)
8378                 goto out;
8379
8380         /* make sure proper unit conversion is applied */
8381         tuned_pa_tactivate =
8382                 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
8383                  / PA_TACTIVATE_TIME_UNIT_US);
8384         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
8385                              tuned_pa_tactivate);
8386
8387 out:
8388         return ret;
8389 }
8390
8391 /**
8392  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
8393  * @hba: per-adapter instance
8394  *
8395  * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
8396  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
8397  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
8398  * This optimal value can help reduce the hibern8 exit latency.
8399  *
8400  * Return: zero on success, non-zero error value on failure.
8401  */
8402 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
8403 {
8404         int ret = 0;
8405         u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
8406         u32 max_hibern8_time, tuned_pa_hibern8time;
8407
8408         ret = ufshcd_dme_get(hba,
8409                              UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
8410                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
8411                                   &local_tx_hibern8_time_cap);
8412         if (ret)
8413                 goto out;
8414
8415         ret = ufshcd_dme_peer_get(hba,
8416                                   UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
8417                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
8418                                   &peer_rx_hibern8_time_cap);
8419         if (ret)
8420                 goto out;
8421
8422         max_hibern8_time = max(local_tx_hibern8_time_cap,
8423                                peer_rx_hibern8_time_cap);
8424         /* make sure proper unit conversion is applied */
8425         tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
8426                                 / PA_HIBERN8_TIME_UNIT_US);
8427         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
8428                              tuned_pa_hibern8time);
8429 out:
8430         return ret;
8431 }
8432
8433 /**
8434  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
8435  * less than device PA_TACTIVATE time.
8436  * @hba: per-adapter instance
8437  *
8438  * Some UFS devices require host PA_TACTIVATE to be lower than device
8439  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
8440  * for such devices.
8441  *
8442  * Return: zero on success, non-zero error value on failure.
8443  */
8444 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
8445 {
8446         int ret = 0;
8447         u32 granularity, peer_granularity;
8448         u32 pa_tactivate, peer_pa_tactivate;
8449         u32 pa_tactivate_us, peer_pa_tactivate_us;
8450         static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
8451
8452         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8453                                   &granularity);
8454         if (ret)
8455                 goto out;
8456
8457         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8458                                   &peer_granularity);
8459         if (ret)
8460                 goto out;
8461
8462         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
8463             (granularity > PA_GRANULARITY_MAX_VAL)) {
8464                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
8465                         __func__, granularity);
8466                 return -EINVAL;
8467         }
8468
8469         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
8470             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
8471                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
8472                         __func__, peer_granularity);
8473                 return -EINVAL;
8474         }
8475
8476         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
8477         if (ret)
8478                 goto out;
8479
8480         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
8481                                   &peer_pa_tactivate);
8482         if (ret)
8483                 goto out;
8484
8485         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
8486         peer_pa_tactivate_us = peer_pa_tactivate *
8487                              gran_to_us_table[peer_granularity - 1];
8488
8489         if (pa_tactivate_us >= peer_pa_tactivate_us) {
8490                 u32 new_peer_pa_tactivate;
8491
8492                 new_peer_pa_tactivate = pa_tactivate_us /
8493                                       gran_to_us_table[peer_granularity - 1];
8494                 new_peer_pa_tactivate++;
8495                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
8496                                           new_peer_pa_tactivate);
8497         }
8498
8499 out:
8500         return ret;
8501 }
8502
8503 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
8504 {
8505         if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
8506                 ufshcd_tune_pa_tactivate(hba);
8507                 ufshcd_tune_pa_hibern8time(hba);
8508         }
8509
8510         ufshcd_vops_apply_dev_quirks(hba);
8511
8512         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
8513                 /* set 1ms timeout for PA_TACTIVATE */
8514                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
8515
8516         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8517                 ufshcd_quirk_tune_host_pa_tactivate(hba);
8518 }
8519
8520 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
8521 {
8522         hba->ufs_stats.hibern8_exit_cnt = 0;
8523         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8524         hba->req_abort_count = 0;
8525 }
8526
8527 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
8528 {
8529         int err;
8530         u8 *desc_buf;
8531
8532         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8533         if (!desc_buf) {
8534                 err = -ENOMEM;
8535                 goto out;
8536         }
8537
8538         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
8539                                      desc_buf, QUERY_DESC_MAX_SIZE);
8540         if (err) {
8541                 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8542                                 __func__, err);
8543                 goto out;
8544         }
8545
8546         if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
8547                 hba->dev_info.max_lu_supported = 32;
8548         else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
8549                 hba->dev_info.max_lu_supported = 8;
8550
8551 out:
8552         kfree(desc_buf);
8553         return err;
8554 }
8555
8556 struct ufs_ref_clk {
8557         unsigned long freq_hz;
8558         enum ufs_ref_clk_freq val;
8559 };
8560
8561 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8562         {19200000, REF_CLK_FREQ_19_2_MHZ},
8563         {26000000, REF_CLK_FREQ_26_MHZ},
8564         {38400000, REF_CLK_FREQ_38_4_MHZ},
8565         {52000000, REF_CLK_FREQ_52_MHZ},
8566         {0, REF_CLK_FREQ_INVAL},
8567 };
8568
8569 static enum ufs_ref_clk_freq
8570 ufs_get_bref_clk_from_hz(unsigned long freq)
8571 {
8572         int i;
8573
8574         for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8575                 if (ufs_ref_clk_freqs[i].freq_hz == freq)
8576                         return ufs_ref_clk_freqs[i].val;
8577
8578         return REF_CLK_FREQ_INVAL;
8579 }
8580
8581 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8582 {
8583         unsigned long freq;
8584
8585         freq = clk_get_rate(refclk);
8586
8587         hba->dev_ref_clk_freq =
8588                 ufs_get_bref_clk_from_hz(freq);
8589
8590         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8591                 dev_err(hba->dev,
8592                 "invalid ref_clk setting = %ld\n", freq);
8593 }
8594
8595 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8596 {
8597         int err;
8598         u32 ref_clk;
8599         u32 freq = hba->dev_ref_clk_freq;
8600
8601         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8602                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8603
8604         if (err) {
8605                 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8606                         err);
8607                 goto out;
8608         }
8609
8610         if (ref_clk == freq)
8611                 goto out; /* nothing to update */
8612
8613         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8614                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8615
8616         if (err) {
8617                 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8618                         ufs_ref_clk_freqs[freq].freq_hz);
8619                 goto out;
8620         }
8621
8622         dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8623                         ufs_ref_clk_freqs[freq].freq_hz);
8624
8625 out:
8626         return err;
8627 }
8628
8629 static int ufshcd_device_params_init(struct ufs_hba *hba)
8630 {
8631         bool flag;
8632         int ret;
8633
8634         /* Init UFS geometry descriptor related parameters */
8635         ret = ufshcd_device_geo_params_init(hba);
8636         if (ret)
8637                 goto out;
8638
8639         /* Check and apply UFS device quirks */
8640         ret = ufs_get_device_desc(hba);
8641         if (ret) {
8642                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8643                         __func__, ret);
8644                 goto out;
8645         }
8646
8647         ufshcd_get_ref_clk_gating_wait(hba);
8648
8649         if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8650                         QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8651                 hba->dev_info.f_power_on_wp_en = flag;
8652
8653         /* Probe maximum power mode co-supported by both UFS host and device */
8654         if (ufshcd_get_max_pwr_mode(hba))
8655                 dev_err(hba->dev,
8656                         "%s: Failed getting max supported power mode\n",
8657                         __func__);
8658 out:
8659         return ret;
8660 }
8661
8662 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba)
8663 {
8664         int err;
8665         struct ufs_query_req *request = NULL;
8666         struct ufs_query_res *response = NULL;
8667         struct ufs_dev_info *dev_info = &hba->dev_info;
8668         struct utp_upiu_query_v4_0 *upiu_data;
8669
8670         if (dev_info->wspecversion < 0x400)
8671                 return;
8672
8673         ufshcd_hold(hba);
8674
8675         mutex_lock(&hba->dev_cmd.lock);
8676
8677         ufshcd_init_query(hba, &request, &response,
8678                           UPIU_QUERY_OPCODE_WRITE_ATTR,
8679                           QUERY_ATTR_IDN_TIMESTAMP, 0, 0);
8680
8681         request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
8682
8683         upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
8684
8685         put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
8686
8687         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
8688
8689         if (err)
8690                 dev_err(hba->dev, "%s: failed to set timestamp %d\n",
8691                         __func__, err);
8692
8693         mutex_unlock(&hba->dev_cmd.lock);
8694         ufshcd_release(hba);
8695 }
8696
8697 /**
8698  * ufshcd_add_lus - probe and add UFS logical units
8699  * @hba: per-adapter instance
8700  *
8701  * Return: 0 upon success; < 0 upon failure.
8702  */
8703 static int ufshcd_add_lus(struct ufs_hba *hba)
8704 {
8705         int ret;
8706
8707         /* Add required well known logical units to scsi mid layer */
8708         ret = ufshcd_scsi_add_wlus(hba);
8709         if (ret)
8710                 goto out;
8711
8712         /* Initialize devfreq after UFS device is detected */
8713         if (ufshcd_is_clkscaling_supported(hba)) {
8714                 memcpy(&hba->clk_scaling.saved_pwr_info,
8715                         &hba->pwr_info,
8716                         sizeof(struct ufs_pa_layer_attr));
8717                 hba->clk_scaling.is_allowed = true;
8718
8719                 ret = ufshcd_devfreq_init(hba);
8720                 if (ret)
8721                         goto out;
8722
8723                 hba->clk_scaling.is_enabled = true;
8724                 ufshcd_init_clk_scaling_sysfs(hba);
8725         }
8726
8727         ufs_bsg_probe(hba);
8728         scsi_scan_host(hba->host);
8729
8730 out:
8731         return ret;
8732 }
8733
8734 /* SDB - Single Doorbell */
8735 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
8736 {
8737         size_t ucdl_size, utrdl_size;
8738
8739         ucdl_size = ufshcd_get_ucd_size(hba) * nutrs;
8740         dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr,
8741                            hba->ucdl_dma_addr);
8742
8743         utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
8744         dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
8745                            hba->utrdl_dma_addr);
8746
8747         devm_kfree(hba->dev, hba->lrb);
8748 }
8749
8750 static int ufshcd_alloc_mcq(struct ufs_hba *hba)
8751 {
8752         int ret;
8753         int old_nutrs = hba->nutrs;
8754
8755         ret = ufshcd_mcq_decide_queue_depth(hba);
8756         if (ret < 0)
8757                 return ret;
8758
8759         hba->nutrs = ret;
8760         ret = ufshcd_mcq_init(hba);
8761         if (ret)
8762                 goto err;
8763
8764         /*
8765          * Previously allocated memory for nutrs may not be enough in MCQ mode.
8766          * Number of supported tags in MCQ mode may be larger than SDB mode.
8767          */
8768         if (hba->nutrs != old_nutrs) {
8769                 ufshcd_release_sdb_queue(hba, old_nutrs);
8770                 ret = ufshcd_memory_alloc(hba);
8771                 if (ret)
8772                         goto err;
8773                 ufshcd_host_memory_configure(hba);
8774         }
8775
8776         ret = ufshcd_mcq_memory_alloc(hba);
8777         if (ret)
8778                 goto err;
8779
8780         return 0;
8781 err:
8782         hba->nutrs = old_nutrs;
8783         return ret;
8784 }
8785
8786 static void ufshcd_config_mcq(struct ufs_hba *hba)
8787 {
8788         int ret;
8789         u32 intrs;
8790
8791         ret = ufshcd_mcq_vops_config_esi(hba);
8792         dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
8793
8794         intrs = UFSHCD_ENABLE_MCQ_INTRS;
8795         if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
8796                 intrs &= ~MCQ_CQ_EVENT_STATUS;
8797         ufshcd_enable_intr(hba, intrs);
8798         ufshcd_mcq_make_queues_operational(hba);
8799         ufshcd_mcq_config_mac(hba, hba->nutrs);
8800
8801         hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
8802         hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
8803
8804         /* Select MCQ mode */
8805         ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1,
8806                       REG_UFS_MEM_CFG);
8807         hba->mcq_enabled = true;
8808
8809         dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n",
8810                  hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT],
8811                  hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL],
8812                  hba->nutrs);
8813 }
8814
8815 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params)
8816 {
8817         int ret;
8818         struct Scsi_Host *host = hba->host;
8819
8820         hba->ufshcd_state = UFSHCD_STATE_RESET;
8821
8822         ret = ufshcd_link_startup(hba);
8823         if (ret)
8824                 return ret;
8825
8826         if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8827                 return ret;
8828
8829         /* Debug counters initialization */
8830         ufshcd_clear_dbg_ufs_stats(hba);
8831
8832         /* UniPro link is active now */
8833         ufshcd_set_link_active(hba);
8834
8835         /* Reconfigure MCQ upon reset */
8836         if (is_mcq_enabled(hba) && !init_dev_params)
8837                 ufshcd_config_mcq(hba);
8838
8839         /* Verify device initialization by sending NOP OUT UPIU */
8840         ret = ufshcd_verify_dev_init(hba);
8841         if (ret)
8842                 return ret;
8843
8844         /* Initiate UFS initialization, and waiting until completion */
8845         ret = ufshcd_complete_dev_init(hba);
8846         if (ret)
8847                 return ret;
8848
8849         /*
8850          * Initialize UFS device parameters used by driver, these
8851          * parameters are associated with UFS descriptors.
8852          */
8853         if (init_dev_params) {
8854                 ret = ufshcd_device_params_init(hba);
8855                 if (ret)
8856                         return ret;
8857                 if (is_mcq_supported(hba) && !hba->scsi_host_added) {
8858                         ret = ufshcd_alloc_mcq(hba);
8859                         if (!ret) {
8860                                 ufshcd_config_mcq(hba);
8861                         } else {
8862                                 /* Continue with SDB mode */
8863                                 use_mcq_mode = false;
8864                                 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n",
8865                                          ret);
8866                         }
8867                         ret = scsi_add_host(host, hba->dev);
8868                         if (ret) {
8869                                 dev_err(hba->dev, "scsi_add_host failed\n");
8870                                 return ret;
8871                         }
8872                         hba->scsi_host_added = true;
8873                 } else if (is_mcq_supported(hba)) {
8874                         /* UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is set */
8875                         ufshcd_config_mcq(hba);
8876                 }
8877         }
8878
8879         ufshcd_tune_unipro_params(hba);
8880
8881         /* UFS device is also active now */
8882         ufshcd_set_ufs_dev_active(hba);
8883         ufshcd_force_reset_auto_bkops(hba);
8884
8885         ufshcd_set_timestamp_attr(hba);
8886         schedule_delayed_work(&hba->ufs_rtc_update_work,
8887                               msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
8888
8889         /* Gear up to HS gear if supported */
8890         if (hba->max_pwr_info.is_valid) {
8891                 /*
8892                  * Set the right value to bRefClkFreq before attempting to
8893                  * switch to HS gears.
8894                  */
8895                 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8896                         ufshcd_set_dev_ref_clk(hba);
8897                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8898                 if (ret) {
8899                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8900                                         __func__, ret);
8901                         return ret;
8902                 }
8903         }
8904
8905         return 0;
8906 }
8907
8908 /**
8909  * ufshcd_probe_hba - probe hba to detect device and initialize it
8910  * @hba: per-adapter instance
8911  * @init_dev_params: whether or not to call ufshcd_device_params_init().
8912  *
8913  * Execute link-startup and verify device initialization
8914  *
8915  * Return: 0 upon success; < 0 upon failure.
8916  */
8917 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8918 {
8919         ktime_t start = ktime_get();
8920         unsigned long flags;
8921         int ret;
8922
8923         ret = ufshcd_device_init(hba, init_dev_params);
8924         if (ret)
8925                 goto out;
8926
8927         if (!hba->pm_op_in_progress &&
8928             (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) {
8929                 /* Reset the device and controller before doing reinit */
8930                 ufshcd_device_reset(hba);
8931                 ufshcd_hba_stop(hba);
8932                 ufshcd_vops_reinit_notify(hba);
8933                 ret = ufshcd_hba_enable(hba);
8934                 if (ret) {
8935                         dev_err(hba->dev, "Host controller enable failed\n");
8936                         ufshcd_print_evt_hist(hba);
8937                         ufshcd_print_host_state(hba);
8938                         goto out;
8939                 }
8940
8941                 /* Reinit the device */
8942                 ret = ufshcd_device_init(hba, init_dev_params);
8943                 if (ret)
8944                         goto out;
8945         }
8946
8947         ufshcd_print_pwr_info(hba);
8948
8949         /*
8950          * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8951          * and for removable UFS card as well, hence always set the parameter.
8952          * Note: Error handler may issue the device reset hence resetting
8953          * bActiveICCLevel as well so it is always safe to set this here.
8954          */
8955         ufshcd_set_active_icc_lvl(hba);
8956
8957         /* Enable UFS Write Booster if supported */
8958         ufshcd_configure_wb(hba);
8959
8960         if (hba->ee_usr_mask)
8961                 ufshcd_write_ee_control(hba);
8962         ufshcd_configure_auto_hibern8(hba);
8963
8964 out:
8965         spin_lock_irqsave(hba->host->host_lock, flags);
8966         if (ret)
8967                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8968         else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8969                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8970         spin_unlock_irqrestore(hba->host->host_lock, flags);
8971
8972         trace_ufshcd_init(dev_name(hba->dev), ret,
8973                 ktime_to_us(ktime_sub(ktime_get(), start)),
8974                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8975         return ret;
8976 }
8977
8978 /**
8979  * ufshcd_async_scan - asynchronous execution for probing hba
8980  * @data: data pointer to pass to this function
8981  * @cookie: cookie data
8982  */
8983 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8984 {
8985         struct ufs_hba *hba = (struct ufs_hba *)data;
8986         int ret;
8987
8988         down(&hba->host_sem);
8989         /* Initialize hba, detect and initialize UFS device */
8990         ret = ufshcd_probe_hba(hba, true);
8991         up(&hba->host_sem);
8992         if (ret)
8993                 goto out;
8994
8995         /* Probe and add UFS logical units  */
8996         ret = ufshcd_add_lus(hba);
8997
8998 out:
8999         pm_runtime_put_sync(hba->dev);
9000
9001         if (ret)
9002                 dev_err(hba->dev, "%s failed: %d\n", __func__, ret);
9003 }
9004
9005 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
9006 {
9007         struct ufs_hba *hba = shost_priv(scmd->device->host);
9008
9009         if (!hba->system_suspending) {
9010                 /* Activate the error handler in the SCSI core. */
9011                 return SCSI_EH_NOT_HANDLED;
9012         }
9013
9014         /*
9015          * If we get here we know that no TMFs are outstanding and also that
9016          * the only pending command is a START STOP UNIT command. Handle the
9017          * timeout of that command directly to prevent a deadlock between
9018          * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler().
9019          */
9020         ufshcd_link_recovery(hba);
9021         dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n",
9022                  __func__, hba->outstanding_tasks);
9023
9024         return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE;
9025 }
9026
9027 static const struct attribute_group *ufshcd_driver_groups[] = {
9028         &ufs_sysfs_unit_descriptor_group,
9029         &ufs_sysfs_lun_attributes_group,
9030         NULL,
9031 };
9032
9033 static struct ufs_hba_variant_params ufs_hba_vps = {
9034         .hba_enable_delay_us            = 1000,
9035         .wb_flush_threshold             = UFS_WB_BUF_REMAIN_PERCENT(40),
9036         .devfreq_profile.polling_ms     = 100,
9037         .devfreq_profile.target         = ufshcd_devfreq_target,
9038         .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
9039         .ondemand_data.upthreshold      = 70,
9040         .ondemand_data.downdifferential = 5,
9041 };
9042
9043 static const struct scsi_host_template ufshcd_driver_template = {
9044         .module                 = THIS_MODULE,
9045         .name                   = UFSHCD,
9046         .proc_name              = UFSHCD,
9047         .map_queues             = ufshcd_map_queues,
9048         .queuecommand           = ufshcd_queuecommand,
9049         .mq_poll                = ufshcd_poll,
9050         .slave_alloc            = ufshcd_slave_alloc,
9051         .slave_configure        = ufshcd_slave_configure,
9052         .slave_destroy          = ufshcd_slave_destroy,
9053         .change_queue_depth     = ufshcd_change_queue_depth,
9054         .eh_abort_handler       = ufshcd_abort,
9055         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
9056         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
9057         .eh_timed_out           = ufshcd_eh_timed_out,
9058         .this_id                = -1,
9059         .sg_tablesize           = SG_ALL,
9060         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
9061         .can_queue              = UFSHCD_CAN_QUEUE,
9062         .max_segment_size       = PRDT_DATA_BYTE_COUNT_MAX,
9063         .max_sectors            = SZ_1M / SECTOR_SIZE,
9064         .max_host_blocked       = 1,
9065         .track_queue_depth      = 1,
9066         .skip_settle_delay      = 1,
9067         .sdev_groups            = ufshcd_driver_groups,
9068         .rpm_autosuspend_delay  = RPM_AUTOSUSPEND_DELAY_MS,
9069 };
9070
9071 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
9072                                    int ua)
9073 {
9074         int ret;
9075
9076         if (!vreg)
9077                 return 0;
9078
9079         /*
9080          * "set_load" operation shall be required on those regulators
9081          * which specifically configured current limitation. Otherwise
9082          * zero max_uA may cause unexpected behavior when regulator is
9083          * enabled or set as high power mode.
9084          */
9085         if (!vreg->max_uA)
9086                 return 0;
9087
9088         ret = regulator_set_load(vreg->reg, ua);
9089         if (ret < 0) {
9090                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
9091                                 __func__, vreg->name, ua, ret);
9092         }
9093
9094         return ret;
9095 }
9096
9097 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
9098                                          struct ufs_vreg *vreg)
9099 {
9100         return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
9101 }
9102
9103 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
9104                                          struct ufs_vreg *vreg)
9105 {
9106         if (!vreg)
9107                 return 0;
9108
9109         return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
9110 }
9111
9112 static int ufshcd_config_vreg(struct device *dev,
9113                 struct ufs_vreg *vreg, bool on)
9114 {
9115         if (regulator_count_voltages(vreg->reg) <= 0)
9116                 return 0;
9117
9118         return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
9119 }
9120
9121 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
9122 {
9123         int ret = 0;
9124
9125         if (!vreg || vreg->enabled)
9126                 goto out;
9127
9128         ret = ufshcd_config_vreg(dev, vreg, true);
9129         if (!ret)
9130                 ret = regulator_enable(vreg->reg);
9131
9132         if (!ret)
9133                 vreg->enabled = true;
9134         else
9135                 dev_err(dev, "%s: %s enable failed, err=%d\n",
9136                                 __func__, vreg->name, ret);
9137 out:
9138         return ret;
9139 }
9140
9141 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
9142 {
9143         int ret = 0;
9144
9145         if (!vreg || !vreg->enabled || vreg->always_on)
9146                 goto out;
9147
9148         ret = regulator_disable(vreg->reg);
9149
9150         if (!ret) {
9151                 /* ignore errors on applying disable config */
9152                 ufshcd_config_vreg(dev, vreg, false);
9153                 vreg->enabled = false;
9154         } else {
9155                 dev_err(dev, "%s: %s disable failed, err=%d\n",
9156                                 __func__, vreg->name, ret);
9157         }
9158 out:
9159         return ret;
9160 }
9161
9162 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
9163 {
9164         int ret = 0;
9165         struct device *dev = hba->dev;
9166         struct ufs_vreg_info *info = &hba->vreg_info;
9167
9168         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
9169         if (ret)
9170                 goto out;
9171
9172         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
9173         if (ret)
9174                 goto out;
9175
9176         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
9177
9178 out:
9179         if (ret) {
9180                 ufshcd_toggle_vreg(dev, info->vccq2, false);
9181                 ufshcd_toggle_vreg(dev, info->vccq, false);
9182                 ufshcd_toggle_vreg(dev, info->vcc, false);
9183         }
9184         return ret;
9185 }
9186
9187 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
9188 {
9189         struct ufs_vreg_info *info = &hba->vreg_info;
9190
9191         return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
9192 }
9193
9194 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
9195 {
9196         int ret = 0;
9197
9198         if (!vreg)
9199                 goto out;
9200
9201         vreg->reg = devm_regulator_get(dev, vreg->name);
9202         if (IS_ERR(vreg->reg)) {
9203                 ret = PTR_ERR(vreg->reg);
9204                 dev_err(dev, "%s: %s get failed, err=%d\n",
9205                                 __func__, vreg->name, ret);
9206         }
9207 out:
9208         return ret;
9209 }
9210 EXPORT_SYMBOL_GPL(ufshcd_get_vreg);
9211
9212 static int ufshcd_init_vreg(struct ufs_hba *hba)
9213 {
9214         int ret = 0;
9215         struct device *dev = hba->dev;
9216         struct ufs_vreg_info *info = &hba->vreg_info;
9217
9218         ret = ufshcd_get_vreg(dev, info->vcc);
9219         if (ret)
9220                 goto out;
9221
9222         ret = ufshcd_get_vreg(dev, info->vccq);
9223         if (!ret)
9224                 ret = ufshcd_get_vreg(dev, info->vccq2);
9225 out:
9226         return ret;
9227 }
9228
9229 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
9230 {
9231         struct ufs_vreg_info *info = &hba->vreg_info;
9232
9233         return ufshcd_get_vreg(hba->dev, info->vdd_hba);
9234 }
9235
9236 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
9237 {
9238         int ret = 0;
9239         struct ufs_clk_info *clki;
9240         struct list_head *head = &hba->clk_list_head;
9241         unsigned long flags;
9242         ktime_t start = ktime_get();
9243         bool clk_state_changed = false;
9244
9245         if (list_empty(head))
9246                 goto out;
9247
9248         ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
9249         if (ret)
9250                 return ret;
9251
9252         list_for_each_entry(clki, head, list) {
9253                 if (!IS_ERR_OR_NULL(clki->clk)) {
9254                         /*
9255                          * Don't disable clocks which are needed
9256                          * to keep the link active.
9257                          */
9258                         if (ufshcd_is_link_active(hba) &&
9259                             clki->keep_link_active)
9260                                 continue;
9261
9262                         clk_state_changed = on ^ clki->enabled;
9263                         if (on && !clki->enabled) {
9264                                 ret = clk_prepare_enable(clki->clk);
9265                                 if (ret) {
9266                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
9267                                                 __func__, clki->name, ret);
9268                                         goto out;
9269                                 }
9270                         } else if (!on && clki->enabled) {
9271                                 clk_disable_unprepare(clki->clk);
9272                         }
9273                         clki->enabled = on;
9274                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
9275                                         clki->name, on ? "en" : "dis");
9276                 }
9277         }
9278
9279         ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
9280         if (ret)
9281                 return ret;
9282
9283 out:
9284         if (ret) {
9285                 list_for_each_entry(clki, head, list) {
9286                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
9287                                 clk_disable_unprepare(clki->clk);
9288                 }
9289         } else if (!ret && on) {
9290                 spin_lock_irqsave(hba->host->host_lock, flags);
9291                 hba->clk_gating.state = CLKS_ON;
9292                 trace_ufshcd_clk_gating(dev_name(hba->dev),
9293                                         hba->clk_gating.state);
9294                 spin_unlock_irqrestore(hba->host->host_lock, flags);
9295         }
9296
9297         if (clk_state_changed)
9298                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
9299                         (on ? "on" : "off"),
9300                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
9301         return ret;
9302 }
9303
9304 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba)
9305 {
9306         u32 freq;
9307         int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
9308
9309         if (ret) {
9310                 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
9311                 return REF_CLK_FREQ_INVAL;
9312         }
9313
9314         return ufs_get_bref_clk_from_hz(freq);
9315 }
9316
9317 static int ufshcd_init_clocks(struct ufs_hba *hba)
9318 {
9319         int ret = 0;
9320         struct ufs_clk_info *clki;
9321         struct device *dev = hba->dev;
9322         struct list_head *head = &hba->clk_list_head;
9323
9324         if (list_empty(head))
9325                 goto out;
9326
9327         list_for_each_entry(clki, head, list) {
9328                 if (!clki->name)
9329                         continue;
9330
9331                 clki->clk = devm_clk_get(dev, clki->name);
9332                 if (IS_ERR(clki->clk)) {
9333                         ret = PTR_ERR(clki->clk);
9334                         dev_err(dev, "%s: %s clk get failed, %d\n",
9335                                         __func__, clki->name, ret);
9336                         goto out;
9337                 }
9338
9339                 /*
9340                  * Parse device ref clk freq as per device tree "ref_clk".
9341                  * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
9342                  * in ufshcd_alloc_host().
9343                  */
9344                 if (!strcmp(clki->name, "ref_clk"))
9345                         ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
9346
9347                 if (clki->max_freq) {
9348                         ret = clk_set_rate(clki->clk, clki->max_freq);
9349                         if (ret) {
9350                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
9351                                         __func__, clki->name,
9352                                         clki->max_freq, ret);
9353                                 goto out;
9354                         }
9355                         clki->curr_freq = clki->max_freq;
9356                 }
9357                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
9358                                 clki->name, clk_get_rate(clki->clk));
9359         }
9360
9361         /* Set Max. frequency for all clocks */
9362         if (hba->use_pm_opp) {
9363                 ret = ufshcd_opp_set_rate(hba, ULONG_MAX);
9364                 if (ret) {
9365                         dev_err(hba->dev, "%s: failed to set OPP: %d", __func__,
9366                                 ret);
9367                         goto out;
9368                 }
9369         }
9370
9371 out:
9372         return ret;
9373 }
9374
9375 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
9376 {
9377         int err = 0;
9378
9379         if (!hba->vops)
9380                 goto out;
9381
9382         err = ufshcd_vops_init(hba);
9383         if (err)
9384                 dev_err_probe(hba->dev, err,
9385                               "%s: variant %s init failed with err %d\n",
9386                               __func__, ufshcd_get_var_name(hba), err);
9387 out:
9388         return err;
9389 }
9390
9391 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
9392 {
9393         if (!hba->vops)
9394                 return;
9395
9396         ufshcd_vops_exit(hba);
9397 }
9398
9399 static int ufshcd_hba_init(struct ufs_hba *hba)
9400 {
9401         int err;
9402
9403         /*
9404          * Handle host controller power separately from the UFS device power
9405          * rails as it will help controlling the UFS host controller power
9406          * collapse easily which is different than UFS device power collapse.
9407          * Also, enable the host controller power before we go ahead with rest
9408          * of the initialization here.
9409          */
9410         err = ufshcd_init_hba_vreg(hba);
9411         if (err)
9412                 goto out;
9413
9414         err = ufshcd_setup_hba_vreg(hba, true);
9415         if (err)
9416                 goto out;
9417
9418         err = ufshcd_init_clocks(hba);
9419         if (err)
9420                 goto out_disable_hba_vreg;
9421
9422         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
9423                 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
9424
9425         err = ufshcd_setup_clocks(hba, true);
9426         if (err)
9427                 goto out_disable_hba_vreg;
9428
9429         err = ufshcd_init_vreg(hba);
9430         if (err)
9431                 goto out_disable_clks;
9432
9433         err = ufshcd_setup_vreg(hba, true);
9434         if (err)
9435                 goto out_disable_clks;
9436
9437         err = ufshcd_variant_hba_init(hba);
9438         if (err)
9439                 goto out_disable_vreg;
9440
9441         ufs_debugfs_hba_init(hba);
9442         ufs_fault_inject_hba_init(hba);
9443
9444         hba->is_powered = true;
9445         goto out;
9446
9447 out_disable_vreg:
9448         ufshcd_setup_vreg(hba, false);
9449 out_disable_clks:
9450         ufshcd_setup_clocks(hba, false);
9451 out_disable_hba_vreg:
9452         ufshcd_setup_hba_vreg(hba, false);
9453 out:
9454         return err;
9455 }
9456
9457 static void ufshcd_hba_exit(struct ufs_hba *hba)
9458 {
9459         if (hba->is_powered) {
9460                 ufshcd_exit_clk_scaling(hba);
9461                 ufshcd_exit_clk_gating(hba);
9462                 if (hba->eh_wq)
9463                         destroy_workqueue(hba->eh_wq);
9464                 ufs_debugfs_hba_exit(hba);
9465                 ufshcd_variant_hba_exit(hba);
9466                 ufshcd_setup_vreg(hba, false);
9467                 ufshcd_setup_clocks(hba, false);
9468                 ufshcd_setup_hba_vreg(hba, false);
9469                 hba->is_powered = false;
9470                 ufs_put_device_desc(hba);
9471         }
9472 }
9473
9474 static int ufshcd_execute_start_stop(struct scsi_device *sdev,
9475                                      enum ufs_dev_pwr_mode pwr_mode,
9476                                      struct scsi_sense_hdr *sshdr)
9477 {
9478         const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 };
9479         const struct scsi_exec_args args = {
9480                 .sshdr = sshdr,
9481                 .req_flags = BLK_MQ_REQ_PM,
9482                 .scmd_flags = SCMD_FAIL_IF_RECOVERING,
9483         };
9484
9485         return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL,
9486                         /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0,
9487                         &args);
9488 }
9489
9490 /**
9491  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
9492  *                           power mode
9493  * @hba: per adapter instance
9494  * @pwr_mode: device power mode to set
9495  *
9496  * Return: 0 if requested power mode is set successfully;
9497  *         < 0 if failed to set the requested power mode.
9498  */
9499 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
9500                                      enum ufs_dev_pwr_mode pwr_mode)
9501 {
9502         struct scsi_sense_hdr sshdr;
9503         struct scsi_device *sdp;
9504         unsigned long flags;
9505         int ret, retries;
9506
9507         spin_lock_irqsave(hba->host->host_lock, flags);
9508         sdp = hba->ufs_device_wlun;
9509         if (sdp && scsi_device_online(sdp))
9510                 ret = scsi_device_get(sdp);
9511         else
9512                 ret = -ENODEV;
9513         spin_unlock_irqrestore(hba->host->host_lock, flags);
9514
9515         if (ret)
9516                 return ret;
9517
9518         /*
9519          * If scsi commands fail, the scsi mid-layer schedules scsi error-
9520          * handling, which would wait for host to be resumed. Since we know
9521          * we are functional while we are here, skip host resume in error
9522          * handling context.
9523          */
9524         hba->host->eh_noresume = 1;
9525
9526         /*
9527          * Current function would be generally called from the power management
9528          * callbacks hence set the RQF_PM flag so that it doesn't resume the
9529          * already suspended childs.
9530          */
9531         for (retries = 3; retries > 0; --retries) {
9532                 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr);
9533                 /*
9534                  * scsi_execute() only returns a negative value if the request
9535                  * queue is dying.
9536                  */
9537                 if (ret <= 0)
9538                         break;
9539         }
9540         if (ret) {
9541                 sdev_printk(KERN_WARNING, sdp,
9542                             "START_STOP failed for power mode: %d, result %x\n",
9543                             pwr_mode, ret);
9544                 if (ret > 0) {
9545                         if (scsi_sense_valid(&sshdr))
9546                                 scsi_print_sense_hdr(sdp, NULL, &sshdr);
9547                         ret = -EIO;
9548                 }
9549         } else {
9550                 hba->curr_dev_pwr_mode = pwr_mode;
9551         }
9552
9553         scsi_device_put(sdp);
9554         hba->host->eh_noresume = 0;
9555         return ret;
9556 }
9557
9558 static int ufshcd_link_state_transition(struct ufs_hba *hba,
9559                                         enum uic_link_state req_link_state,
9560                                         bool check_for_bkops)
9561 {
9562         int ret = 0;
9563
9564         if (req_link_state == hba->uic_link_state)
9565                 return 0;
9566
9567         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
9568                 ret = ufshcd_uic_hibern8_enter(hba);
9569                 if (!ret) {
9570                         ufshcd_set_link_hibern8(hba);
9571                 } else {
9572                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9573                                         __func__, ret);
9574                         goto out;
9575                 }
9576         }
9577         /*
9578          * If autobkops is enabled, link can't be turned off because
9579          * turning off the link would also turn off the device, except in the
9580          * case of DeepSleep where the device is expected to remain powered.
9581          */
9582         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
9583                  (!check_for_bkops || !hba->auto_bkops_enabled)) {
9584                 /*
9585                  * Let's make sure that link is in low power mode, we are doing
9586                  * this currently by putting the link in Hibern8. Otherway to
9587                  * put the link in low power mode is to send the DME end point
9588                  * to device and then send the DME reset command to local
9589                  * unipro. But putting the link in hibern8 is much faster.
9590                  *
9591                  * Note also that putting the link in Hibern8 is a requirement
9592                  * for entering DeepSleep.
9593                  */
9594                 ret = ufshcd_uic_hibern8_enter(hba);
9595                 if (ret) {
9596                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9597                                         __func__, ret);
9598                         goto out;
9599                 }
9600                 /*
9601                  * Change controller state to "reset state" which
9602                  * should also put the link in off/reset state
9603                  */
9604                 ufshcd_hba_stop(hba);
9605                 /*
9606                  * TODO: Check if we need any delay to make sure that
9607                  * controller is reset
9608                  */
9609                 ufshcd_set_link_off(hba);
9610         }
9611
9612 out:
9613         return ret;
9614 }
9615
9616 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
9617 {
9618         bool vcc_off = false;
9619
9620         /*
9621          * It seems some UFS devices may keep drawing more than sleep current
9622          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
9623          * To avoid this situation, add 2ms delay before putting these UFS
9624          * rails in LPM mode.
9625          */
9626         if (!ufshcd_is_link_active(hba) &&
9627             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
9628                 usleep_range(2000, 2100);
9629
9630         /*
9631          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
9632          * power.
9633          *
9634          * If UFS device and link is in OFF state, all power supplies (VCC,
9635          * VCCQ, VCCQ2) can be turned off if power on write protect is not
9636          * required. If UFS link is inactive (Hibern8 or OFF state) and device
9637          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
9638          *
9639          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
9640          * in low power state which would save some power.
9641          *
9642          * If Write Booster is enabled and the device needs to flush the WB
9643          * buffer OR if bkops status is urgent for WB, keep Vcc on.
9644          */
9645         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9646             !hba->dev_info.is_lu_power_on_wp) {
9647                 ufshcd_setup_vreg(hba, false);
9648                 vcc_off = true;
9649         } else if (!ufshcd_is_ufs_dev_active(hba)) {
9650                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9651                 vcc_off = true;
9652                 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
9653                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9654                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
9655                 }
9656         }
9657
9658         /*
9659          * Some UFS devices require delay after VCC power rail is turned-off.
9660          */
9661         if (vcc_off && hba->vreg_info.vcc &&
9662                 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
9663                 usleep_range(5000, 5100);
9664 }
9665
9666 #ifdef CONFIG_PM
9667 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
9668 {
9669         int ret = 0;
9670
9671         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9672             !hba->dev_info.is_lu_power_on_wp) {
9673                 ret = ufshcd_setup_vreg(hba, true);
9674         } else if (!ufshcd_is_ufs_dev_active(hba)) {
9675                 if (!ufshcd_is_link_active(hba)) {
9676                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
9677                         if (ret)
9678                                 goto vcc_disable;
9679                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
9680                         if (ret)
9681                                 goto vccq_lpm;
9682                 }
9683                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
9684         }
9685         goto out;
9686
9687 vccq_lpm:
9688         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9689 vcc_disable:
9690         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9691 out:
9692         return ret;
9693 }
9694 #endif /* CONFIG_PM */
9695
9696 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
9697 {
9698         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9699                 ufshcd_setup_hba_vreg(hba, false);
9700 }
9701
9702 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
9703 {
9704         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9705                 ufshcd_setup_hba_vreg(hba, true);
9706 }
9707
9708 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9709 {
9710         int ret = 0;
9711         bool check_for_bkops;
9712         enum ufs_pm_level pm_lvl;
9713         enum ufs_dev_pwr_mode req_dev_pwr_mode;
9714         enum uic_link_state req_link_state;
9715
9716         hba->pm_op_in_progress = true;
9717         if (pm_op != UFS_SHUTDOWN_PM) {
9718                 pm_lvl = pm_op == UFS_RUNTIME_PM ?
9719                          hba->rpm_lvl : hba->spm_lvl;
9720                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
9721                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
9722         } else {
9723                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
9724                 req_link_state = UIC_LINK_OFF_STATE;
9725         }
9726
9727         /*
9728          * If we can't transition into any of the low power modes
9729          * just gate the clocks.
9730          */
9731         ufshcd_hold(hba);
9732         hba->clk_gating.is_suspended = true;
9733
9734         if (ufshcd_is_clkscaling_supported(hba))
9735                 ufshcd_clk_scaling_suspend(hba, true);
9736
9737         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
9738                         req_link_state == UIC_LINK_ACTIVE_STATE) {
9739                 goto vops_suspend;
9740         }
9741
9742         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9743             (req_link_state == hba->uic_link_state))
9744                 goto enable_scaling;
9745
9746         /* UFS device & link must be active before we enter in this function */
9747         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
9748                 ret = -EINVAL;
9749                 goto enable_scaling;
9750         }
9751
9752         if (pm_op == UFS_RUNTIME_PM) {
9753                 if (ufshcd_can_autobkops_during_suspend(hba)) {
9754                         /*
9755                          * The device is idle with no requests in the queue,
9756                          * allow background operations if bkops status shows
9757                          * that performance might be impacted.
9758                          */
9759                         ret = ufshcd_urgent_bkops(hba);
9760                         if (ret) {
9761                                 /*
9762                                  * If return err in suspend flow, IO will hang.
9763                                  * Trigger error handler and break suspend for
9764                                  * error recovery.
9765                                  */
9766                                 ufshcd_force_error_recovery(hba);
9767                                 ret = -EBUSY;
9768                                 goto enable_scaling;
9769                         }
9770                 } else {
9771                         /* make sure that auto bkops is disabled */
9772                         ufshcd_disable_auto_bkops(hba);
9773                 }
9774                 /*
9775                  * If device needs to do BKOP or WB buffer flush during
9776                  * Hibern8, keep device power mode as "active power mode"
9777                  * and VCC supply.
9778                  */
9779                 hba->dev_info.b_rpm_dev_flush_capable =
9780                         hba->auto_bkops_enabled ||
9781                         (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
9782                         ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
9783                         ufshcd_is_auto_hibern8_enabled(hba))) &&
9784                         ufshcd_wb_need_flush(hba));
9785         }
9786
9787         flush_work(&hba->eeh_work);
9788
9789         ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9790         if (ret)
9791                 goto enable_scaling;
9792
9793         if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9794                 if (pm_op != UFS_RUNTIME_PM)
9795                         /* ensure that bkops is disabled */
9796                         ufshcd_disable_auto_bkops(hba);
9797
9798                 if (!hba->dev_info.b_rpm_dev_flush_capable) {
9799                         ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9800                         if (ret && pm_op != UFS_SHUTDOWN_PM) {
9801                                 /*
9802                                  * If return err in suspend flow, IO will hang.
9803                                  * Trigger error handler and break suspend for
9804                                  * error recovery.
9805                                  */
9806                                 ufshcd_force_error_recovery(hba);
9807                                 ret = -EBUSY;
9808                         }
9809                         if (ret)
9810                                 goto enable_scaling;
9811                 }
9812         }
9813
9814         /*
9815          * In the case of DeepSleep, the device is expected to remain powered
9816          * with the link off, so do not check for bkops.
9817          */
9818         check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9819         ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9820         if (ret && pm_op != UFS_SHUTDOWN_PM) {
9821                 /*
9822                  * If return err in suspend flow, IO will hang.
9823                  * Trigger error handler and break suspend for
9824                  * error recovery.
9825                  */
9826                 ufshcd_force_error_recovery(hba);
9827                 ret = -EBUSY;
9828         }
9829         if (ret)
9830                 goto set_dev_active;
9831
9832 vops_suspend:
9833         /*
9834          * Call vendor specific suspend callback. As these callbacks may access
9835          * vendor specific host controller register space call them before the
9836          * host clocks are ON.
9837          */
9838         ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9839         if (ret)
9840                 goto set_link_active;
9841
9842         cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
9843         goto out;
9844
9845 set_link_active:
9846         /*
9847          * Device hardware reset is required to exit DeepSleep. Also, for
9848          * DeepSleep, the link is off so host reset and restore will be done
9849          * further below.
9850          */
9851         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9852                 ufshcd_device_reset(hba);
9853                 WARN_ON(!ufshcd_is_link_off(hba));
9854         }
9855         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9856                 ufshcd_set_link_active(hba);
9857         else if (ufshcd_is_link_off(hba))
9858                 ufshcd_host_reset_and_restore(hba);
9859 set_dev_active:
9860         /* Can also get here needing to exit DeepSleep */
9861         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9862                 ufshcd_device_reset(hba);
9863                 ufshcd_host_reset_and_restore(hba);
9864         }
9865         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9866                 ufshcd_disable_auto_bkops(hba);
9867 enable_scaling:
9868         if (ufshcd_is_clkscaling_supported(hba))
9869                 ufshcd_clk_scaling_suspend(hba, false);
9870
9871         hba->dev_info.b_rpm_dev_flush_capable = false;
9872 out:
9873         if (hba->dev_info.b_rpm_dev_flush_capable) {
9874                 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9875                         msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9876         }
9877
9878         if (ret) {
9879                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9880                 hba->clk_gating.is_suspended = false;
9881                 ufshcd_release(hba);
9882         }
9883         hba->pm_op_in_progress = false;
9884         return ret;
9885 }
9886
9887 #ifdef CONFIG_PM
9888 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9889 {
9890         int ret;
9891         enum uic_link_state old_link_state = hba->uic_link_state;
9892
9893         hba->pm_op_in_progress = true;
9894
9895         /*
9896          * Call vendor specific resume callback. As these callbacks may access
9897          * vendor specific host controller register space call them when the
9898          * host clocks are ON.
9899          */
9900         ret = ufshcd_vops_resume(hba, pm_op);
9901         if (ret)
9902                 goto out;
9903
9904         /* For DeepSleep, the only supported option is to have the link off */
9905         WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9906
9907         if (ufshcd_is_link_hibern8(hba)) {
9908                 ret = ufshcd_uic_hibern8_exit(hba);
9909                 if (!ret) {
9910                         ufshcd_set_link_active(hba);
9911                 } else {
9912                         dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9913                                         __func__, ret);
9914                         goto vendor_suspend;
9915                 }
9916         } else if (ufshcd_is_link_off(hba)) {
9917                 /*
9918                  * A full initialization of the host and the device is
9919                  * required since the link was put to off during suspend.
9920                  * Note, in the case of DeepSleep, the device will exit
9921                  * DeepSleep due to device reset.
9922                  */
9923                 ret = ufshcd_reset_and_restore(hba);
9924                 /*
9925                  * ufshcd_reset_and_restore() should have already
9926                  * set the link state as active
9927                  */
9928                 if (ret || !ufshcd_is_link_active(hba))
9929                         goto vendor_suspend;
9930         }
9931
9932         if (!ufshcd_is_ufs_dev_active(hba)) {
9933                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9934                 if (ret)
9935                         goto set_old_link_state;
9936                 ufshcd_set_timestamp_attr(hba);
9937                 schedule_delayed_work(&hba->ufs_rtc_update_work,
9938                                       msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
9939         }
9940
9941         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9942                 ufshcd_enable_auto_bkops(hba);
9943         else
9944                 /*
9945                  * If BKOPs operations are urgently needed at this moment then
9946                  * keep auto-bkops enabled or else disable it.
9947                  */
9948                 ufshcd_urgent_bkops(hba);
9949
9950         if (hba->ee_usr_mask)
9951                 ufshcd_write_ee_control(hba);
9952
9953         if (ufshcd_is_clkscaling_supported(hba))
9954                 ufshcd_clk_scaling_suspend(hba, false);
9955
9956         if (hba->dev_info.b_rpm_dev_flush_capable) {
9957                 hba->dev_info.b_rpm_dev_flush_capable = false;
9958                 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9959         }
9960
9961         ufshcd_configure_auto_hibern8(hba);
9962
9963         goto out;
9964
9965 set_old_link_state:
9966         ufshcd_link_state_transition(hba, old_link_state, 0);
9967 vendor_suspend:
9968         ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9969         ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9970 out:
9971         if (ret)
9972                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9973         hba->clk_gating.is_suspended = false;
9974         ufshcd_release(hba);
9975         hba->pm_op_in_progress = false;
9976         return ret;
9977 }
9978
9979 static int ufshcd_wl_runtime_suspend(struct device *dev)
9980 {
9981         struct scsi_device *sdev = to_scsi_device(dev);
9982         struct ufs_hba *hba;
9983         int ret;
9984         ktime_t start = ktime_get();
9985
9986         hba = shost_priv(sdev->host);
9987
9988         ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9989         if (ret)
9990                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9991
9992         trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9993                 ktime_to_us(ktime_sub(ktime_get(), start)),
9994                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9995
9996         return ret;
9997 }
9998
9999 static int ufshcd_wl_runtime_resume(struct device *dev)
10000 {
10001         struct scsi_device *sdev = to_scsi_device(dev);
10002         struct ufs_hba *hba;
10003         int ret = 0;
10004         ktime_t start = ktime_get();
10005
10006         hba = shost_priv(sdev->host);
10007
10008         ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
10009         if (ret)
10010                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
10011
10012         trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
10013                 ktime_to_us(ktime_sub(ktime_get(), start)),
10014                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10015
10016         return ret;
10017 }
10018 #endif
10019
10020 #ifdef CONFIG_PM_SLEEP
10021 static int ufshcd_wl_suspend(struct device *dev)
10022 {
10023         struct scsi_device *sdev = to_scsi_device(dev);
10024         struct ufs_hba *hba;
10025         int ret = 0;
10026         ktime_t start = ktime_get();
10027
10028         hba = shost_priv(sdev->host);
10029         down(&hba->host_sem);
10030         hba->system_suspending = true;
10031
10032         if (pm_runtime_suspended(dev))
10033                 goto out;
10034
10035         ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
10036         if (ret) {
10037                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
10038                 up(&hba->host_sem);
10039         }
10040
10041 out:
10042         if (!ret)
10043                 hba->is_sys_suspended = true;
10044         trace_ufshcd_wl_suspend(dev_name(dev), ret,
10045                 ktime_to_us(ktime_sub(ktime_get(), start)),
10046                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10047
10048         return ret;
10049 }
10050
10051 static int ufshcd_wl_resume(struct device *dev)
10052 {
10053         struct scsi_device *sdev = to_scsi_device(dev);
10054         struct ufs_hba *hba;
10055         int ret = 0;
10056         ktime_t start = ktime_get();
10057
10058         hba = shost_priv(sdev->host);
10059
10060         if (pm_runtime_suspended(dev))
10061                 goto out;
10062
10063         ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
10064         if (ret)
10065                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
10066 out:
10067         trace_ufshcd_wl_resume(dev_name(dev), ret,
10068                 ktime_to_us(ktime_sub(ktime_get(), start)),
10069                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10070         if (!ret)
10071                 hba->is_sys_suspended = false;
10072         hba->system_suspending = false;
10073         up(&hba->host_sem);
10074         return ret;
10075 }
10076 #endif
10077
10078 /**
10079  * ufshcd_suspend - helper function for suspend operations
10080  * @hba: per adapter instance
10081  *
10082  * This function will put disable irqs, turn off clocks
10083  * and set vreg and hba-vreg in lpm mode.
10084  *
10085  * Return: 0 upon success; < 0 upon failure.
10086  */
10087 static int ufshcd_suspend(struct ufs_hba *hba)
10088 {
10089         int ret;
10090
10091         if (!hba->is_powered)
10092                 return 0;
10093         /*
10094          * Disable the host irq as host controller as there won't be any
10095          * host controller transaction expected till resume.
10096          */
10097         ufshcd_disable_irq(hba);
10098         ret = ufshcd_setup_clocks(hba, false);
10099         if (ret) {
10100                 ufshcd_enable_irq(hba);
10101                 return ret;
10102         }
10103         if (ufshcd_is_clkgating_allowed(hba)) {
10104                 hba->clk_gating.state = CLKS_OFF;
10105                 trace_ufshcd_clk_gating(dev_name(hba->dev),
10106                                         hba->clk_gating.state);
10107         }
10108
10109         ufshcd_vreg_set_lpm(hba);
10110         /* Put the host controller in low power mode if possible */
10111         ufshcd_hba_vreg_set_lpm(hba);
10112         return ret;
10113 }
10114
10115 #ifdef CONFIG_PM
10116 /**
10117  * ufshcd_resume - helper function for resume operations
10118  * @hba: per adapter instance
10119  *
10120  * This function basically turns on the regulators, clocks and
10121  * irqs of the hba.
10122  *
10123  * Return: 0 for success and non-zero for failure.
10124  */
10125 static int ufshcd_resume(struct ufs_hba *hba)
10126 {
10127         int ret;
10128
10129         if (!hba->is_powered)
10130                 return 0;
10131
10132         ufshcd_hba_vreg_set_hpm(hba);
10133         ret = ufshcd_vreg_set_hpm(hba);
10134         if (ret)
10135                 goto out;
10136
10137         /* Make sure clocks are enabled before accessing controller */
10138         ret = ufshcd_setup_clocks(hba, true);
10139         if (ret)
10140                 goto disable_vreg;
10141
10142         /* enable the host irq as host controller would be active soon */
10143         ufshcd_enable_irq(hba);
10144
10145         goto out;
10146
10147 disable_vreg:
10148         ufshcd_vreg_set_lpm(hba);
10149 out:
10150         if (ret)
10151                 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
10152         return ret;
10153 }
10154 #endif /* CONFIG_PM */
10155
10156 #ifdef CONFIG_PM_SLEEP
10157 /**
10158  * ufshcd_system_suspend - system suspend callback
10159  * @dev: Device associated with the UFS controller.
10160  *
10161  * Executed before putting the system into a sleep state in which the contents
10162  * of main memory are preserved.
10163  *
10164  * Return: 0 for success and non-zero for failure.
10165  */
10166 int ufshcd_system_suspend(struct device *dev)
10167 {
10168         struct ufs_hba *hba = dev_get_drvdata(dev);
10169         int ret = 0;
10170         ktime_t start = ktime_get();
10171
10172         if (pm_runtime_suspended(hba->dev))
10173                 goto out;
10174
10175         ret = ufshcd_suspend(hba);
10176 out:
10177         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
10178                 ktime_to_us(ktime_sub(ktime_get(), start)),
10179                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10180         return ret;
10181 }
10182 EXPORT_SYMBOL(ufshcd_system_suspend);
10183
10184 /**
10185  * ufshcd_system_resume - system resume callback
10186  * @dev: Device associated with the UFS controller.
10187  *
10188  * Executed after waking the system up from a sleep state in which the contents
10189  * of main memory were preserved.
10190  *
10191  * Return: 0 for success and non-zero for failure.
10192  */
10193 int ufshcd_system_resume(struct device *dev)
10194 {
10195         struct ufs_hba *hba = dev_get_drvdata(dev);
10196         ktime_t start = ktime_get();
10197         int ret = 0;
10198
10199         if (pm_runtime_suspended(hba->dev))
10200                 goto out;
10201
10202         ret = ufshcd_resume(hba);
10203
10204 out:
10205         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
10206                 ktime_to_us(ktime_sub(ktime_get(), start)),
10207                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10208
10209         return ret;
10210 }
10211 EXPORT_SYMBOL(ufshcd_system_resume);
10212 #endif /* CONFIG_PM_SLEEP */
10213
10214 #ifdef CONFIG_PM
10215 /**
10216  * ufshcd_runtime_suspend - runtime suspend callback
10217  * @dev: Device associated with the UFS controller.
10218  *
10219  * Check the description of ufshcd_suspend() function for more details.
10220  *
10221  * Return: 0 for success and non-zero for failure.
10222  */
10223 int ufshcd_runtime_suspend(struct device *dev)
10224 {
10225         struct ufs_hba *hba = dev_get_drvdata(dev);
10226         int ret;
10227         ktime_t start = ktime_get();
10228
10229         ret = ufshcd_suspend(hba);
10230
10231         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
10232                 ktime_to_us(ktime_sub(ktime_get(), start)),
10233                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10234         return ret;
10235 }
10236 EXPORT_SYMBOL(ufshcd_runtime_suspend);
10237
10238 /**
10239  * ufshcd_runtime_resume - runtime resume routine
10240  * @dev: Device associated with the UFS controller.
10241  *
10242  * This function basically brings controller
10243  * to active state. Following operations are done in this function:
10244  *
10245  * 1. Turn on all the controller related clocks
10246  * 2. Turn ON VCC rail
10247  *
10248  * Return: 0 upon success; < 0 upon failure.
10249  */
10250 int ufshcd_runtime_resume(struct device *dev)
10251 {
10252         struct ufs_hba *hba = dev_get_drvdata(dev);
10253         int ret;
10254         ktime_t start = ktime_get();
10255
10256         ret = ufshcd_resume(hba);
10257
10258         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
10259                 ktime_to_us(ktime_sub(ktime_get(), start)),
10260                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10261         return ret;
10262 }
10263 EXPORT_SYMBOL(ufshcd_runtime_resume);
10264 #endif /* CONFIG_PM */
10265
10266 static void ufshcd_wl_shutdown(struct device *dev)
10267 {
10268         struct scsi_device *sdev = to_scsi_device(dev);
10269         struct ufs_hba *hba = shost_priv(sdev->host);
10270
10271         down(&hba->host_sem);
10272         hba->shutting_down = true;
10273         up(&hba->host_sem);
10274
10275         /* Turn on everything while shutting down */
10276         ufshcd_rpm_get_sync(hba);
10277         scsi_device_quiesce(sdev);
10278         shost_for_each_device(sdev, hba->host) {
10279                 if (sdev == hba->ufs_device_wlun)
10280                         continue;
10281                 scsi_device_quiesce(sdev);
10282         }
10283         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10284
10285         /*
10286          * Next, turn off the UFS controller and the UFS regulators. Disable
10287          * clocks.
10288          */
10289         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
10290                 ufshcd_suspend(hba);
10291
10292         hba->is_powered = false;
10293 }
10294
10295 /**
10296  * ufshcd_remove - de-allocate SCSI host and host memory space
10297  *              data structure memory
10298  * @hba: per adapter instance
10299  */
10300 void ufshcd_remove(struct ufs_hba *hba)
10301 {
10302         if (hba->ufs_device_wlun)
10303                 ufshcd_rpm_get_sync(hba);
10304         ufs_hwmon_remove(hba);
10305         ufs_bsg_remove(hba);
10306         ufs_sysfs_remove_nodes(hba->dev);
10307         blk_mq_destroy_queue(hba->tmf_queue);
10308         blk_put_queue(hba->tmf_queue);
10309         blk_mq_free_tag_set(&hba->tmf_tag_set);
10310         scsi_remove_host(hba->host);
10311         /* disable interrupts */
10312         ufshcd_disable_intr(hba, hba->intr_mask);
10313         ufshcd_hba_stop(hba);
10314         ufshcd_hba_exit(hba);
10315 }
10316 EXPORT_SYMBOL_GPL(ufshcd_remove);
10317
10318 #ifdef CONFIG_PM_SLEEP
10319 int ufshcd_system_freeze(struct device *dev)
10320 {
10321
10322         return ufshcd_system_suspend(dev);
10323
10324 }
10325 EXPORT_SYMBOL_GPL(ufshcd_system_freeze);
10326
10327 int ufshcd_system_restore(struct device *dev)
10328 {
10329
10330         struct ufs_hba *hba = dev_get_drvdata(dev);
10331         int ret;
10332
10333         ret = ufshcd_system_resume(dev);
10334         if (ret)
10335                 return ret;
10336
10337         /* Configure UTRL and UTMRL base address registers */
10338         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
10339                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
10340         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
10341                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
10342         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
10343                         REG_UTP_TASK_REQ_LIST_BASE_L);
10344         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
10345                         REG_UTP_TASK_REQ_LIST_BASE_H);
10346         /*
10347          * Make sure that UTRL and UTMRL base address registers
10348          * are updated with the latest queue addresses. Only after
10349          * updating these addresses, we can queue the new commands.
10350          */
10351         mb();
10352
10353         /* Resuming from hibernate, assume that link was OFF */
10354         ufshcd_set_link_off(hba);
10355
10356         return 0;
10357
10358 }
10359 EXPORT_SYMBOL_GPL(ufshcd_system_restore);
10360
10361 int ufshcd_system_thaw(struct device *dev)
10362 {
10363         return ufshcd_system_resume(dev);
10364 }
10365 EXPORT_SYMBOL_GPL(ufshcd_system_thaw);
10366 #endif /* CONFIG_PM_SLEEP  */
10367
10368 /**
10369  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
10370  * @hba: pointer to Host Bus Adapter (HBA)
10371  */
10372 void ufshcd_dealloc_host(struct ufs_hba *hba)
10373 {
10374         scsi_host_put(hba->host);
10375 }
10376 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
10377
10378 /**
10379  * ufshcd_set_dma_mask - Set dma mask based on the controller
10380  *                       addressing capability
10381  * @hba: per adapter instance
10382  *
10383  * Return: 0 for success, non-zero for failure.
10384  */
10385 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
10386 {
10387         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
10388                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
10389                         return 0;
10390         }
10391         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
10392 }
10393
10394 /**
10395  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
10396  * @dev: pointer to device handle
10397  * @hba_handle: driver private handle
10398  *
10399  * Return: 0 on success, non-zero value on failure.
10400  */
10401 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
10402 {
10403         struct Scsi_Host *host;
10404         struct ufs_hba *hba;
10405         int err = 0;
10406
10407         if (!dev) {
10408                 dev_err(dev,
10409                 "Invalid memory reference for dev is NULL\n");
10410                 err = -ENODEV;
10411                 goto out_error;
10412         }
10413
10414         host = scsi_host_alloc(&ufshcd_driver_template,
10415                                 sizeof(struct ufs_hba));
10416         if (!host) {
10417                 dev_err(dev, "scsi_host_alloc failed\n");
10418                 err = -ENOMEM;
10419                 goto out_error;
10420         }
10421         host->nr_maps = HCTX_TYPE_POLL + 1;
10422         hba = shost_priv(host);
10423         hba->host = host;
10424         hba->dev = dev;
10425         hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
10426         hba->nop_out_timeout = NOP_OUT_TIMEOUT;
10427         ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry));
10428         INIT_LIST_HEAD(&hba->clk_list_head);
10429         spin_lock_init(&hba->outstanding_lock);
10430
10431         *hba_handle = hba;
10432
10433 out_error:
10434         return err;
10435 }
10436 EXPORT_SYMBOL(ufshcd_alloc_host);
10437
10438 /* This function exists because blk_mq_alloc_tag_set() requires this. */
10439 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
10440                                      const struct blk_mq_queue_data *qd)
10441 {
10442         WARN_ON_ONCE(true);
10443         return BLK_STS_NOTSUPP;
10444 }
10445
10446 static const struct blk_mq_ops ufshcd_tmf_ops = {
10447         .queue_rq = ufshcd_queue_tmf,
10448 };
10449
10450 /**
10451  * ufshcd_init - Driver initialization routine
10452  * @hba: per-adapter instance
10453  * @mmio_base: base register address
10454  * @irq: Interrupt line of device
10455  *
10456  * Return: 0 on success, non-zero value on failure.
10457  */
10458 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
10459 {
10460         int err;
10461         struct Scsi_Host *host = hba->host;
10462         struct device *dev = hba->dev;
10463         char eh_wq_name[sizeof("ufs_eh_wq_00")];
10464
10465         /*
10466          * dev_set_drvdata() must be called before any callbacks are registered
10467          * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
10468          * sysfs).
10469          */
10470         dev_set_drvdata(dev, hba);
10471
10472         if (!mmio_base) {
10473                 dev_err(hba->dev,
10474                 "Invalid memory reference for mmio_base is NULL\n");
10475                 err = -ENODEV;
10476                 goto out_error;
10477         }
10478
10479         hba->mmio_base = mmio_base;
10480         hba->irq = irq;
10481         hba->vps = &ufs_hba_vps;
10482
10483         err = ufshcd_hba_init(hba);
10484         if (err)
10485                 goto out_error;
10486
10487         /* Read capabilities registers */
10488         err = ufshcd_hba_capabilities(hba);
10489         if (err)
10490                 goto out_disable;
10491
10492         /* Get UFS version supported by the controller */
10493         hba->ufs_version = ufshcd_get_ufs_version(hba);
10494
10495         /* Get Interrupt bit mask per version */
10496         hba->intr_mask = ufshcd_get_intr_mask(hba);
10497
10498         err = ufshcd_set_dma_mask(hba);
10499         if (err) {
10500                 dev_err(hba->dev, "set dma mask failed\n");
10501                 goto out_disable;
10502         }
10503
10504         /* Allocate memory for host memory space */
10505         err = ufshcd_memory_alloc(hba);
10506         if (err) {
10507                 dev_err(hba->dev, "Memory allocation failed\n");
10508                 goto out_disable;
10509         }
10510
10511         /* Configure LRB */
10512         ufshcd_host_memory_configure(hba);
10513
10514         host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
10515         host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
10516         host->max_id = UFSHCD_MAX_ID;
10517         host->max_lun = UFS_MAX_LUNS;
10518         host->max_channel = UFSHCD_MAX_CHANNEL;
10519         host->unique_id = host->host_no;
10520         host->max_cmd_len = UFS_CDB_SIZE;
10521         host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
10522
10523         hba->max_pwr_info.is_valid = false;
10524
10525         /* Initialize work queues */
10526         snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
10527                  hba->host->host_no);
10528         hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
10529         if (!hba->eh_wq) {
10530                 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
10531                         __func__);
10532                 err = -ENOMEM;
10533                 goto out_disable;
10534         }
10535         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
10536         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
10537
10538         sema_init(&hba->host_sem, 1);
10539
10540         /* Initialize UIC command mutex */
10541         mutex_init(&hba->uic_cmd_mutex);
10542
10543         /* Initialize mutex for device management commands */
10544         mutex_init(&hba->dev_cmd.lock);
10545
10546         /* Initialize mutex for exception event control */
10547         mutex_init(&hba->ee_ctrl_mutex);
10548
10549         mutex_init(&hba->wb_mutex);
10550         init_rwsem(&hba->clk_scaling_lock);
10551
10552         ufshcd_init_clk_gating(hba);
10553
10554         ufshcd_init_clk_scaling(hba);
10555
10556         /*
10557          * In order to avoid any spurious interrupt immediately after
10558          * registering UFS controller interrupt handler, clear any pending UFS
10559          * interrupt status and disable all the UFS interrupts.
10560          */
10561         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
10562                       REG_INTERRUPT_STATUS);
10563         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
10564         /*
10565          * Make sure that UFS interrupts are disabled and any pending interrupt
10566          * status is cleared before registering UFS interrupt handler.
10567          */
10568         mb();
10569
10570         /* IRQ registration */
10571         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
10572         if (err) {
10573                 dev_err(hba->dev, "request irq failed\n");
10574                 goto out_disable;
10575         } else {
10576                 hba->is_irq_enabled = true;
10577         }
10578
10579         if (!is_mcq_supported(hba)) {
10580                 err = scsi_add_host(host, hba->dev);
10581                 if (err) {
10582                         dev_err(hba->dev, "scsi_add_host failed\n");
10583                         goto out_disable;
10584                 }
10585         }
10586
10587         hba->tmf_tag_set = (struct blk_mq_tag_set) {
10588                 .nr_hw_queues   = 1,
10589                 .queue_depth    = hba->nutmrs,
10590                 .ops            = &ufshcd_tmf_ops,
10591                 .flags          = BLK_MQ_F_NO_SCHED,
10592         };
10593         err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
10594         if (err < 0)
10595                 goto out_remove_scsi_host;
10596         hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL);
10597         if (IS_ERR(hba->tmf_queue)) {
10598                 err = PTR_ERR(hba->tmf_queue);
10599                 goto free_tmf_tag_set;
10600         }
10601         hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
10602                                     sizeof(*hba->tmf_rqs), GFP_KERNEL);
10603         if (!hba->tmf_rqs) {
10604                 err = -ENOMEM;
10605                 goto free_tmf_queue;
10606         }
10607
10608         /* Reset the attached device */
10609         ufshcd_device_reset(hba);
10610
10611         ufshcd_init_crypto(hba);
10612
10613         /* Host controller enable */
10614         err = ufshcd_hba_enable(hba);
10615         if (err) {
10616                 dev_err(hba->dev, "Host controller enable failed\n");
10617                 ufshcd_print_evt_hist(hba);
10618                 ufshcd_print_host_state(hba);
10619                 goto free_tmf_queue;
10620         }
10621
10622         /*
10623          * Set the default power management level for runtime and system PM.
10624          * Default power saving mode is to keep UFS link in Hibern8 state
10625          * and UFS device in sleep state.
10626          */
10627         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10628                                                 UFS_SLEEP_PWR_MODE,
10629                                                 UIC_LINK_HIBERN8_STATE);
10630         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10631                                                 UFS_SLEEP_PWR_MODE,
10632                                                 UIC_LINK_HIBERN8_STATE);
10633
10634         INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work);
10635         INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work);
10636
10637         /* Set the default auto-hiberate idle timer value to 150 ms */
10638         if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
10639                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
10640                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
10641         }
10642
10643         /* Hold auto suspend until async scan completes */
10644         pm_runtime_get_sync(dev);
10645         atomic_set(&hba->scsi_block_reqs_cnt, 0);
10646         /*
10647          * We are assuming that device wasn't put in sleep/power-down
10648          * state exclusively during the boot stage before kernel.
10649          * This assumption helps avoid doing link startup twice during
10650          * ufshcd_probe_hba().
10651          */
10652         ufshcd_set_ufs_dev_active(hba);
10653
10654         async_schedule(ufshcd_async_scan, hba);
10655         ufs_sysfs_add_nodes(hba->dev);
10656
10657         device_enable_async_suspend(dev);
10658         return 0;
10659
10660 free_tmf_queue:
10661         blk_mq_destroy_queue(hba->tmf_queue);
10662         blk_put_queue(hba->tmf_queue);
10663 free_tmf_tag_set:
10664         blk_mq_free_tag_set(&hba->tmf_tag_set);
10665 out_remove_scsi_host:
10666         scsi_remove_host(hba->host);
10667 out_disable:
10668         hba->is_irq_enabled = false;
10669         ufshcd_hba_exit(hba);
10670 out_error:
10671         return err;
10672 }
10673 EXPORT_SYMBOL_GPL(ufshcd_init);
10674
10675 void ufshcd_resume_complete(struct device *dev)
10676 {
10677         struct ufs_hba *hba = dev_get_drvdata(dev);
10678
10679         if (hba->complete_put) {
10680                 ufshcd_rpm_put(hba);
10681                 hba->complete_put = false;
10682         }
10683 }
10684 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
10685
10686 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
10687 {
10688         struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
10689         enum ufs_dev_pwr_mode dev_pwr_mode;
10690         enum uic_link_state link_state;
10691         unsigned long flags;
10692         bool res;
10693
10694         spin_lock_irqsave(&dev->power.lock, flags);
10695         dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
10696         link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
10697         res = pm_runtime_suspended(dev) &&
10698               hba->curr_dev_pwr_mode == dev_pwr_mode &&
10699               hba->uic_link_state == link_state &&
10700               !hba->dev_info.b_rpm_dev_flush_capable;
10701         spin_unlock_irqrestore(&dev->power.lock, flags);
10702
10703         return res;
10704 }
10705
10706 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
10707 {
10708         struct ufs_hba *hba = dev_get_drvdata(dev);
10709         int ret;
10710
10711         /*
10712          * SCSI assumes that runtime-pm and system-pm for scsi drivers
10713          * are same. And it doesn't wake up the device for system-suspend
10714          * if it's runtime suspended. But ufs doesn't follow that.
10715          * Refer ufshcd_resume_complete()
10716          */
10717         if (hba->ufs_device_wlun) {
10718                 /* Prevent runtime suspend */
10719                 ufshcd_rpm_get_noresume(hba);
10720                 /*
10721                  * Check if already runtime suspended in same state as system
10722                  * suspend would be.
10723                  */
10724                 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
10725                         /* RPM state is not ok for SPM, so runtime resume */
10726                         ret = ufshcd_rpm_resume(hba);
10727                         if (ret < 0 && ret != -EACCES) {
10728                                 ufshcd_rpm_put(hba);
10729                                 return ret;
10730                         }
10731                 }
10732                 hba->complete_put = true;
10733         }
10734         return 0;
10735 }
10736 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
10737
10738 int ufshcd_suspend_prepare(struct device *dev)
10739 {
10740         return __ufshcd_suspend_prepare(dev, true);
10741 }
10742 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
10743
10744 #ifdef CONFIG_PM_SLEEP
10745 static int ufshcd_wl_poweroff(struct device *dev)
10746 {
10747         struct scsi_device *sdev = to_scsi_device(dev);
10748         struct ufs_hba *hba = shost_priv(sdev->host);
10749
10750         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10751         return 0;
10752 }
10753 #endif
10754
10755 static int ufshcd_wl_probe(struct device *dev)
10756 {
10757         struct scsi_device *sdev = to_scsi_device(dev);
10758
10759         if (!is_device_wlun(sdev))
10760                 return -ENODEV;
10761
10762         blk_pm_runtime_init(sdev->request_queue, dev);
10763         pm_runtime_set_autosuspend_delay(dev, 0);
10764         pm_runtime_allow(dev);
10765
10766         return  0;
10767 }
10768
10769 static int ufshcd_wl_remove(struct device *dev)
10770 {
10771         pm_runtime_forbid(dev);
10772         return 0;
10773 }
10774
10775 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
10776 #ifdef CONFIG_PM_SLEEP
10777         .suspend = ufshcd_wl_suspend,
10778         .resume = ufshcd_wl_resume,
10779         .freeze = ufshcd_wl_suspend,
10780         .thaw = ufshcd_wl_resume,
10781         .poweroff = ufshcd_wl_poweroff,
10782         .restore = ufshcd_wl_resume,
10783 #endif
10784         SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
10785 };
10786
10787 static void ufshcd_check_header_layout(void)
10788 {
10789         /*
10790          * gcc compilers before version 10 cannot do constant-folding for
10791          * sub-byte bitfields. Hence skip the layout checks for gcc 9 and
10792          * before.
10793          */
10794         if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000)
10795                 return;
10796
10797         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10798                                 .cci = 3})[0] != 3);
10799
10800         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10801                                 .ehs_length = 2})[1] != 2);
10802
10803         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10804                                 .enable_crypto = 1})[2]
10805                      != 0x80);
10806
10807         BUILD_BUG_ON((((u8 *)&(struct request_desc_header){
10808                                         .command_type = 5,
10809                                         .data_direction = 3,
10810                                         .interrupt = 1,
10811                                 })[3]) != ((5 << 4) | (3 << 1) | 1));
10812
10813         BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10814                                 .dunl = cpu_to_le32(0xdeadbeef)})[1] !=
10815                 cpu_to_le32(0xdeadbeef));
10816
10817         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10818                                 .ocs = 4})[8] != 4);
10819
10820         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10821                                 .cds = 5})[9] != 5);
10822
10823         BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10824                                 .dunu = cpu_to_le32(0xbadcafe)})[3] !=
10825                 cpu_to_le32(0xbadcafe));
10826
10827         BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10828                              .iid = 0xf })[4] != 0xf0);
10829
10830         BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10831                              .command_set_type = 0xf })[4] != 0xf);
10832 }
10833
10834 /*
10835  * ufs_dev_wlun_template - describes ufs device wlun
10836  * ufs-device wlun - used to send pm commands
10837  * All luns are consumers of ufs-device wlun.
10838  *
10839  * Currently, no sd driver is present for wluns.
10840  * Hence the no specific pm operations are performed.
10841  * With ufs design, SSU should be sent to ufs-device wlun.
10842  * Hence register a scsi driver for ufs wluns only.
10843  */
10844 static struct scsi_driver ufs_dev_wlun_template = {
10845         .gendrv = {
10846                 .name = "ufs_device_wlun",
10847                 .owner = THIS_MODULE,
10848                 .probe = ufshcd_wl_probe,
10849                 .remove = ufshcd_wl_remove,
10850                 .pm = &ufshcd_wl_pm_ops,
10851                 .shutdown = ufshcd_wl_shutdown,
10852         },
10853 };
10854
10855 static int __init ufshcd_core_init(void)
10856 {
10857         int ret;
10858
10859         ufshcd_check_header_layout();
10860
10861         ufs_debugfs_init();
10862
10863         ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
10864         if (ret)
10865                 ufs_debugfs_exit();
10866         return ret;
10867 }
10868
10869 static void __exit ufshcd_core_exit(void)
10870 {
10871         ufs_debugfs_exit();
10872         scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
10873 }
10874
10875 module_init(ufshcd_core_init);
10876 module_exit(ufshcd_core_exit);
10877
10878 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
10879 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
10880 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
10881 MODULE_SOFTDEP("pre: governor_simpleondemand");
10882 MODULE_LICENSE("GPL");