Revert "scsi: ufs: disable vccq if it's not needed by UFS device"
[sfrench/cifs-2.6.git] / drivers / scsi / ufs / ufshcd.c
1 /*
2  * Universal Flash Storage Host controller driver Core
3  *
4  * This code is based on drivers/scsi/ufs/ufshcd.c
5  * Copyright (C) 2011-2013 Samsung India Software Operations
6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7  *
8  * Authors:
9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
10  *      Vinayak Holikatti <h.vinayak@samsung.com>
11  *
12  * This program is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU General Public License
14  * as published by the Free Software Foundation; either version 2
15  * of the License, or (at your option) any later version.
16  * See the COPYING file in the top-level directory or visit
17  * <http://www.gnu.org/licenses/gpl-2.0.html>
18  *
19  * This program is distributed in the hope that it will be useful,
20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22  * GNU General Public License for more details.
23  *
24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
25  * without warranty of any kind. You are solely responsible for
26  * determining the appropriateness of using and distributing
27  * the program and assume all risks associated with your exercise
28  * of rights with respect to the program, including but not limited
29  * to infringement of third party rights, the risks and costs of
30  * program errors, damage to or loss of data, programs or equipment,
31  * and unavailability or interruption of operations. Under no
32  * circumstances will the contributor of this Program be liable for
33  * any damages of any kind arising from your use or distribution of
34  * this program.
35  *
36  * The Linux Foundation chooses to take subject only to the GPLv2
37  * license terms, and distributes only under these terms.
38  */
39
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
45 #include "ufshcd.h"
46 #include "ufs_quirks.h"
47 #include "unipro.h"
48 #include "ufs-sysfs.h"
49 #include "ufs_bsg.h"
50
51 #define CREATE_TRACE_POINTS
52 #include <trace/events/ufs.h>
53
54 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
55                                  UTP_TASK_REQ_COMPL |\
56                                  UFSHCD_ERROR_MASK)
57 /* UIC command timeout, unit: ms */
58 #define UIC_CMD_TIMEOUT 500
59
60 /* NOP OUT retries waiting for NOP IN response */
61 #define NOP_OUT_RETRIES    10
62 /* Timeout after 30 msecs if NOP OUT hangs without response */
63 #define NOP_OUT_TIMEOUT    30 /* msecs */
64
65 /* Query request retries */
66 #define QUERY_REQ_RETRIES 3
67 /* Query request timeout */
68 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
69
70 /* Task management command timeout */
71 #define TM_CMD_TIMEOUT  100 /* msecs */
72
73 /* maximum number of retries for a general UIC command  */
74 #define UFS_UIC_COMMAND_RETRIES 3
75
76 /* maximum number of link-startup retries */
77 #define DME_LINKSTARTUP_RETRIES 3
78
79 /* Maximum retries for Hibern8 enter */
80 #define UIC_HIBERN8_ENTER_RETRIES 3
81
82 /* maximum number of reset retries before giving up */
83 #define MAX_HOST_RESET_RETRIES 5
84
85 /* Expose the flag value from utp_upiu_query.value */
86 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
88 /* Interrupt aggregation default timeout, unit: 40us */
89 #define INT_AGGR_DEF_TO 0x02
90
91 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
92         ({                                                              \
93                 int _ret;                                               \
94                 if (_on)                                                \
95                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
96                 else                                                    \
97                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
98                 _ret;                                                   \
99         })
100
101 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
102         size_t __len = (len);                                            \
103         print_hex_dump(KERN_ERR, prefix_str,                             \
104                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
105                        16, 4, buf, __len, false);                        \
106 } while (0)
107
108 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
109                      const char *prefix)
110 {
111         u8 *regs;
112
113         regs = kzalloc(len, GFP_KERNEL);
114         if (!regs)
115                 return -ENOMEM;
116
117         memcpy_fromio(regs, hba->mmio_base + offset, len);
118         ufshcd_hex_dump(prefix, regs, len);
119         kfree(regs);
120
121         return 0;
122 }
123 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
124
125 enum {
126         UFSHCD_MAX_CHANNEL      = 0,
127         UFSHCD_MAX_ID           = 1,
128         UFSHCD_CMD_PER_LUN      = 32,
129         UFSHCD_CAN_QUEUE        = 32,
130 };
131
132 /* UFSHCD states */
133 enum {
134         UFSHCD_STATE_RESET,
135         UFSHCD_STATE_ERROR,
136         UFSHCD_STATE_OPERATIONAL,
137         UFSHCD_STATE_EH_SCHEDULED,
138 };
139
140 /* UFSHCD error handling flags */
141 enum {
142         UFSHCD_EH_IN_PROGRESS = (1 << 0),
143 };
144
145 /* UFSHCD UIC layer error flags */
146 enum {
147         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
148         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
149         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
150         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
151         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
152         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
153 };
154
155 #define ufshcd_set_eh_in_progress(h) \
156         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
157 #define ufshcd_eh_in_progress(h) \
158         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
159 #define ufshcd_clear_eh_in_progress(h) \
160         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
161
162 #define ufshcd_set_ufs_dev_active(h) \
163         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
164 #define ufshcd_set_ufs_dev_sleep(h) \
165         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
166 #define ufshcd_set_ufs_dev_poweroff(h) \
167         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
168 #define ufshcd_is_ufs_dev_active(h) \
169         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
170 #define ufshcd_is_ufs_dev_sleep(h) \
171         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
172 #define ufshcd_is_ufs_dev_poweroff(h) \
173         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
174
175 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
176         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
177         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
178         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
179         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
180         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
181         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
182 };
183
184 static inline enum ufs_dev_pwr_mode
185 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
186 {
187         return ufs_pm_lvl_states[lvl].dev_state;
188 }
189
190 static inline enum uic_link_state
191 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
192 {
193         return ufs_pm_lvl_states[lvl].link_state;
194 }
195
196 static inline enum ufs_pm_level
197 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
198                                         enum uic_link_state link_state)
199 {
200         enum ufs_pm_level lvl;
201
202         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
203                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
204                         (ufs_pm_lvl_states[lvl].link_state == link_state))
205                         return lvl;
206         }
207
208         /* if no match found, return the level 0 */
209         return UFS_PM_LVL_0;
210 }
211
212 static struct ufs_dev_fix ufs_fixups[] = {
213         /* UFS cards deviations table */
214         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
215                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
216         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
217         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
218                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
219         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
220                 UFS_DEVICE_NO_FASTAUTO),
221         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
222                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
223         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
224                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
225         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
226                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
227         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
228                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
229         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
230         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
231                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
232         UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
233                 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
234
235         END_FIX
236 };
237
238 static void ufshcd_tmc_handler(struct ufs_hba *hba);
239 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
240 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
241 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
242 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
243 static void ufshcd_hba_exit(struct ufs_hba *hba);
244 static int ufshcd_probe_hba(struct ufs_hba *hba);
245 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
246                                  bool skip_ref_clk);
247 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
248 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
249 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
250 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
251 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
252 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
253 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
254 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
255 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
256 static irqreturn_t ufshcd_intr(int irq, void *__hba);
257 static int ufshcd_change_power_mode(struct ufs_hba *hba,
258                              struct ufs_pa_layer_attr *pwr_mode);
259 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
260 {
261         return tag >= 0 && tag < hba->nutrs;
262 }
263
264 static inline int ufshcd_enable_irq(struct ufs_hba *hba)
265 {
266         int ret = 0;
267
268         if (!hba->is_irq_enabled) {
269                 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
270                                 hba);
271                 if (ret)
272                         dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
273                                 __func__, ret);
274                 hba->is_irq_enabled = true;
275         }
276
277         return ret;
278 }
279
280 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
281 {
282         if (hba->is_irq_enabled) {
283                 free_irq(hba->irq, hba);
284                 hba->is_irq_enabled = false;
285         }
286 }
287
288 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
289 {
290         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
291                 scsi_unblock_requests(hba->host);
292 }
293
294 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
295 {
296         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
297                 scsi_block_requests(hba->host);
298 }
299
300 /* replace non-printable or non-ASCII characters with spaces */
301 static inline void ufshcd_remove_non_printable(char *val)
302 {
303         if (!val)
304                 return;
305
306         if (*val < 0x20 || *val > 0x7e)
307                 *val = ' ';
308 }
309
310 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
311                 const char *str)
312 {
313         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
314
315         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
316 }
317
318 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
319                 const char *str)
320 {
321         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
322
323         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
324 }
325
326 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
327                 const char *str)
328 {
329         int off = (int)tag - hba->nutrs;
330         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
331
332         trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
333                         &descp->input_param1);
334 }
335
336 static void ufshcd_add_command_trace(struct ufs_hba *hba,
337                 unsigned int tag, const char *str)
338 {
339         sector_t lba = -1;
340         u8 opcode = 0;
341         u32 intr, doorbell;
342         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
343         int transfer_len = -1;
344
345         if (!trace_ufshcd_command_enabled()) {
346                 /* trace UPIU W/O tracing command */
347                 if (lrbp->cmd)
348                         ufshcd_add_cmd_upiu_trace(hba, tag, str);
349                 return;
350         }
351
352         if (lrbp->cmd) { /* data phase exists */
353                 /* trace UPIU also */
354                 ufshcd_add_cmd_upiu_trace(hba, tag, str);
355                 opcode = (u8)(*lrbp->cmd->cmnd);
356                 if ((opcode == READ_10) || (opcode == WRITE_10)) {
357                         /*
358                          * Currently we only fully trace read(10) and write(10)
359                          * commands
360                          */
361                         if (lrbp->cmd->request && lrbp->cmd->request->bio)
362                                 lba =
363                                   lrbp->cmd->request->bio->bi_iter.bi_sector;
364                         transfer_len = be32_to_cpu(
365                                 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
366                 }
367         }
368
369         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
370         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
371         trace_ufshcd_command(dev_name(hba->dev), str, tag,
372                                 doorbell, transfer_len, intr, lba, opcode);
373 }
374
375 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
376 {
377         struct ufs_clk_info *clki;
378         struct list_head *head = &hba->clk_list_head;
379
380         if (list_empty(head))
381                 return;
382
383         list_for_each_entry(clki, head, list) {
384                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
385                                 clki->max_freq)
386                         dev_err(hba->dev, "clk: %s, rate: %u\n",
387                                         clki->name, clki->curr_freq);
388         }
389 }
390
391 static void ufshcd_print_uic_err_hist(struct ufs_hba *hba,
392                 struct ufs_uic_err_reg_hist *err_hist, char *err_name)
393 {
394         int i;
395         bool found = false;
396
397         for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) {
398                 int p = (i + err_hist->pos) % UIC_ERR_REG_HIST_LENGTH;
399
400                 if (err_hist->reg[p] == 0)
401                         continue;
402                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i,
403                         err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
404                 found = true;
405         }
406
407         if (!found)
408                 dev_err(hba->dev, "No record of %s uic errors\n", err_name);
409 }
410
411 static void ufshcd_print_host_regs(struct ufs_hba *hba)
412 {
413         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
414         dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
415                 hba->ufs_version, hba->capabilities);
416         dev_err(hba->dev,
417                 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
418                 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
419         dev_err(hba->dev,
420                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
421                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
422                 hba->ufs_stats.hibern8_exit_cnt);
423
424         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
425         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
426         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
427         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
428         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
429
430         ufshcd_print_clk_freqs(hba);
431
432         if (hba->vops && hba->vops->dbg_register_dump)
433                 hba->vops->dbg_register_dump(hba);
434 }
435
436 static
437 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
438 {
439         struct ufshcd_lrb *lrbp;
440         int prdt_length;
441         int tag;
442
443         for_each_set_bit(tag, &bitmap, hba->nutrs) {
444                 lrbp = &hba->lrb[tag];
445
446                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
447                                 tag, ktime_to_us(lrbp->issue_time_stamp));
448                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
449                                 tag, ktime_to_us(lrbp->compl_time_stamp));
450                 dev_err(hba->dev,
451                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
452                         tag, (u64)lrbp->utrd_dma_addr);
453
454                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
455                                 sizeof(struct utp_transfer_req_desc));
456                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
457                         (u64)lrbp->ucd_req_dma_addr);
458                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
459                                 sizeof(struct utp_upiu_req));
460                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
461                         (u64)lrbp->ucd_rsp_dma_addr);
462                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
463                                 sizeof(struct utp_upiu_rsp));
464
465                 prdt_length = le16_to_cpu(
466                         lrbp->utr_descriptor_ptr->prd_table_length);
467                 dev_err(hba->dev,
468                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
469                         tag, prdt_length,
470                         (u64)lrbp->ucd_prdt_dma_addr);
471
472                 if (pr_prdt)
473                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
474                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
475         }
476 }
477
478 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
479 {
480         int tag;
481
482         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
483                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
484
485                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
486                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
487         }
488 }
489
490 static void ufshcd_print_host_state(struct ufs_hba *hba)
491 {
492         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
493         dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
494                 hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
495         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
496                 hba->saved_err, hba->saved_uic_err);
497         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
498                 hba->curr_dev_pwr_mode, hba->uic_link_state);
499         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
500                 hba->pm_op_in_progress, hba->is_sys_suspended);
501         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
502                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
503         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
504         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
505                 hba->eh_flags, hba->req_abort_count);
506         dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
507                 hba->capabilities, hba->caps);
508         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
509                 hba->dev_quirks);
510 }
511
512 /**
513  * ufshcd_print_pwr_info - print power params as saved in hba
514  * power info
515  * @hba: per-adapter instance
516  */
517 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
518 {
519         static const char * const names[] = {
520                 "INVALID MODE",
521                 "FAST MODE",
522                 "SLOW_MODE",
523                 "INVALID MODE",
524                 "FASTAUTO_MODE",
525                 "SLOWAUTO_MODE",
526                 "INVALID MODE",
527         };
528
529         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
530                  __func__,
531                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
532                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
533                  names[hba->pwr_info.pwr_rx],
534                  names[hba->pwr_info.pwr_tx],
535                  hba->pwr_info.hs_rate);
536 }
537
538 /*
539  * ufshcd_wait_for_register - wait for register value to change
540  * @hba - per-adapter interface
541  * @reg - mmio register offset
542  * @mask - mask to apply to read register value
543  * @val - wait condition
544  * @interval_us - polling interval in microsecs
545  * @timeout_ms - timeout in millisecs
546  * @can_sleep - perform sleep or just spin
547  *
548  * Returns -ETIMEDOUT on error, zero on success
549  */
550 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
551                                 u32 val, unsigned long interval_us,
552                                 unsigned long timeout_ms, bool can_sleep)
553 {
554         int err = 0;
555         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
556
557         /* ignore bits that we don't intend to wait on */
558         val = val & mask;
559
560         while ((ufshcd_readl(hba, reg) & mask) != val) {
561                 if (can_sleep)
562                         usleep_range(interval_us, interval_us + 50);
563                 else
564                         udelay(interval_us);
565                 if (time_after(jiffies, timeout)) {
566                         if ((ufshcd_readl(hba, reg) & mask) != val)
567                                 err = -ETIMEDOUT;
568                         break;
569                 }
570         }
571
572         return err;
573 }
574
575 /**
576  * ufshcd_get_intr_mask - Get the interrupt bit mask
577  * @hba: Pointer to adapter instance
578  *
579  * Returns interrupt bit mask per version
580  */
581 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
582 {
583         u32 intr_mask = 0;
584
585         switch (hba->ufs_version) {
586         case UFSHCI_VERSION_10:
587                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
588                 break;
589         case UFSHCI_VERSION_11:
590         case UFSHCI_VERSION_20:
591                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
592                 break;
593         case UFSHCI_VERSION_21:
594         default:
595                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
596                 break;
597         }
598
599         return intr_mask;
600 }
601
602 /**
603  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
604  * @hba: Pointer to adapter instance
605  *
606  * Returns UFSHCI version supported by the controller
607  */
608 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
609 {
610         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
611                 return ufshcd_vops_get_ufs_hci_version(hba);
612
613         return ufshcd_readl(hba, REG_UFS_VERSION);
614 }
615
616 /**
617  * ufshcd_is_device_present - Check if any device connected to
618  *                            the host controller
619  * @hba: pointer to adapter instance
620  *
621  * Returns true if device present, false if no device detected
622  */
623 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
624 {
625         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
626                                                 DEVICE_PRESENT) ? true : false;
627 }
628
629 /**
630  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
631  * @lrbp: pointer to local command reference block
632  *
633  * This function is used to get the OCS field from UTRD
634  * Returns the OCS field in the UTRD
635  */
636 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
637 {
638         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
639 }
640
641 /**
642  * ufshcd_get_tm_free_slot - get a free slot for task management request
643  * @hba: per adapter instance
644  * @free_slot: pointer to variable with available slot value
645  *
646  * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
647  * Returns 0 if free slot is not available, else return 1 with tag value
648  * in @free_slot.
649  */
650 static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
651 {
652         int tag;
653         bool ret = false;
654
655         if (!free_slot)
656                 goto out;
657
658         do {
659                 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
660                 if (tag >= hba->nutmrs)
661                         goto out;
662         } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
663
664         *free_slot = tag;
665         ret = true;
666 out:
667         return ret;
668 }
669
670 static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
671 {
672         clear_bit_unlock(slot, &hba->tm_slots_in_use);
673 }
674
675 /**
676  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
677  * @hba: per adapter instance
678  * @pos: position of the bit to be cleared
679  */
680 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
681 {
682         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
683                 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
684         else
685                 ufshcd_writel(hba, ~(1 << pos),
686                                 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
687 }
688
689 /**
690  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
691  * @hba: per adapter instance
692  * @pos: position of the bit to be cleared
693  */
694 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
695 {
696         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
697                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
698         else
699                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
700 }
701
702 /**
703  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
704  * @hba: per adapter instance
705  * @tag: position of the bit to be cleared
706  */
707 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
708 {
709         __clear_bit(tag, &hba->outstanding_reqs);
710 }
711
712 /**
713  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
714  * @reg: Register value of host controller status
715  *
716  * Returns integer, 0 on Success and positive value if failed
717  */
718 static inline int ufshcd_get_lists_status(u32 reg)
719 {
720         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
721 }
722
723 /**
724  * ufshcd_get_uic_cmd_result - Get the UIC command result
725  * @hba: Pointer to adapter instance
726  *
727  * This function gets the result of UIC command completion
728  * Returns 0 on success, non zero value on error
729  */
730 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
731 {
732         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
733                MASK_UIC_COMMAND_RESULT;
734 }
735
736 /**
737  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
738  * @hba: Pointer to adapter instance
739  *
740  * This function gets UIC command argument3
741  * Returns 0 on success, non zero value on error
742  */
743 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
744 {
745         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
746 }
747
748 /**
749  * ufshcd_get_req_rsp - returns the TR response transaction type
750  * @ucd_rsp_ptr: pointer to response UPIU
751  */
752 static inline int
753 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
754 {
755         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
756 }
757
758 /**
759  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
760  * @ucd_rsp_ptr: pointer to response UPIU
761  *
762  * This function gets the response status and scsi_status from response UPIU
763  * Returns the response result code.
764  */
765 static inline int
766 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
767 {
768         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
769 }
770
771 /*
772  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
773  *                              from response UPIU
774  * @ucd_rsp_ptr: pointer to response UPIU
775  *
776  * Return the data segment length.
777  */
778 static inline unsigned int
779 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
780 {
781         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
782                 MASK_RSP_UPIU_DATA_SEG_LEN;
783 }
784
785 /**
786  * ufshcd_is_exception_event - Check if the device raised an exception event
787  * @ucd_rsp_ptr: pointer to response UPIU
788  *
789  * The function checks if the device raised an exception event indicated in
790  * the Device Information field of response UPIU.
791  *
792  * Returns true if exception is raised, false otherwise.
793  */
794 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
795 {
796         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
797                         MASK_RSP_EXCEPTION_EVENT ? true : false;
798 }
799
800 /**
801  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
802  * @hba: per adapter instance
803  */
804 static inline void
805 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
806 {
807         ufshcd_writel(hba, INT_AGGR_ENABLE |
808                       INT_AGGR_COUNTER_AND_TIMER_RESET,
809                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
810 }
811
812 /**
813  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
814  * @hba: per adapter instance
815  * @cnt: Interrupt aggregation counter threshold
816  * @tmout: Interrupt aggregation timeout value
817  */
818 static inline void
819 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
820 {
821         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
822                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
823                       INT_AGGR_TIMEOUT_VAL(tmout),
824                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
825 }
826
827 /**
828  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
829  * @hba: per adapter instance
830  */
831 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
832 {
833         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
834 }
835
836 /**
837  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
838  *                      When run-stop registers are set to 1, it indicates the
839  *                      host controller that it can process the requests
840  * @hba: per adapter instance
841  */
842 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
843 {
844         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
845                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
846         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
847                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
848 }
849
850 /**
851  * ufshcd_hba_start - Start controller initialization sequence
852  * @hba: per adapter instance
853  */
854 static inline void ufshcd_hba_start(struct ufs_hba *hba)
855 {
856         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
857 }
858
859 /**
860  * ufshcd_is_hba_active - Get controller state
861  * @hba: per adapter instance
862  *
863  * Returns false if controller is active, true otherwise
864  */
865 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
866 {
867         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
868                 ? false : true;
869 }
870
871 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
872 {
873         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
874         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
875             (hba->ufs_version == UFSHCI_VERSION_11))
876                 return UFS_UNIPRO_VER_1_41;
877         else
878                 return UFS_UNIPRO_VER_1_6;
879 }
880 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
881
882 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
883 {
884         /*
885          * If both host and device support UniPro ver1.6 or later, PA layer
886          * parameters tuning happens during link startup itself.
887          *
888          * We can manually tune PA layer parameters if either host or device
889          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
890          * logic simple, we will only do manual tuning if local unipro version
891          * doesn't support ver1.6 or later.
892          */
893         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
894                 return true;
895         else
896                 return false;
897 }
898
899 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
900 {
901         int ret = 0;
902         struct ufs_clk_info *clki;
903         struct list_head *head = &hba->clk_list_head;
904         ktime_t start = ktime_get();
905         bool clk_state_changed = false;
906
907         if (list_empty(head))
908                 goto out;
909
910         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
911         if (ret)
912                 return ret;
913
914         list_for_each_entry(clki, head, list) {
915                 if (!IS_ERR_OR_NULL(clki->clk)) {
916                         if (scale_up && clki->max_freq) {
917                                 if (clki->curr_freq == clki->max_freq)
918                                         continue;
919
920                                 clk_state_changed = true;
921                                 ret = clk_set_rate(clki->clk, clki->max_freq);
922                                 if (ret) {
923                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
924                                                 __func__, clki->name,
925                                                 clki->max_freq, ret);
926                                         break;
927                                 }
928                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
929                                                 "scaled up", clki->name,
930                                                 clki->curr_freq,
931                                                 clki->max_freq);
932
933                                 clki->curr_freq = clki->max_freq;
934
935                         } else if (!scale_up && clki->min_freq) {
936                                 if (clki->curr_freq == clki->min_freq)
937                                         continue;
938
939                                 clk_state_changed = true;
940                                 ret = clk_set_rate(clki->clk, clki->min_freq);
941                                 if (ret) {
942                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
943                                                 __func__, clki->name,
944                                                 clki->min_freq, ret);
945                                         break;
946                                 }
947                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
948                                                 "scaled down", clki->name,
949                                                 clki->curr_freq,
950                                                 clki->min_freq);
951                                 clki->curr_freq = clki->min_freq;
952                         }
953                 }
954                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
955                                 clki->name, clk_get_rate(clki->clk));
956         }
957
958         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
959
960 out:
961         if (clk_state_changed)
962                 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
963                         (scale_up ? "up" : "down"),
964                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
965         return ret;
966 }
967
968 /**
969  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
970  * @hba: per adapter instance
971  * @scale_up: True if scaling up and false if scaling down
972  *
973  * Returns true if scaling is required, false otherwise.
974  */
975 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
976                                                bool scale_up)
977 {
978         struct ufs_clk_info *clki;
979         struct list_head *head = &hba->clk_list_head;
980
981         if (list_empty(head))
982                 return false;
983
984         list_for_each_entry(clki, head, list) {
985                 if (!IS_ERR_OR_NULL(clki->clk)) {
986                         if (scale_up && clki->max_freq) {
987                                 if (clki->curr_freq == clki->max_freq)
988                                         continue;
989                                 return true;
990                         } else if (!scale_up && clki->min_freq) {
991                                 if (clki->curr_freq == clki->min_freq)
992                                         continue;
993                                 return true;
994                         }
995                 }
996         }
997
998         return false;
999 }
1000
1001 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1002                                         u64 wait_timeout_us)
1003 {
1004         unsigned long flags;
1005         int ret = 0;
1006         u32 tm_doorbell;
1007         u32 tr_doorbell;
1008         bool timeout = false, do_last_check = false;
1009         ktime_t start;
1010
1011         ufshcd_hold(hba, false);
1012         spin_lock_irqsave(hba->host->host_lock, flags);
1013         /*
1014          * Wait for all the outstanding tasks/transfer requests.
1015          * Verify by checking the doorbell registers are clear.
1016          */
1017         start = ktime_get();
1018         do {
1019                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1020                         ret = -EBUSY;
1021                         goto out;
1022                 }
1023
1024                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1025                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1026                 if (!tm_doorbell && !tr_doorbell) {
1027                         timeout = false;
1028                         break;
1029                 } else if (do_last_check) {
1030                         break;
1031                 }
1032
1033                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1034                 schedule();
1035                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1036                     wait_timeout_us) {
1037                         timeout = true;
1038                         /*
1039                          * We might have scheduled out for long time so make
1040                          * sure to check if doorbells are cleared by this time
1041                          * or not.
1042                          */
1043                         do_last_check = true;
1044                 }
1045                 spin_lock_irqsave(hba->host->host_lock, flags);
1046         } while (tm_doorbell || tr_doorbell);
1047
1048         if (timeout) {
1049                 dev_err(hba->dev,
1050                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1051                         __func__, tm_doorbell, tr_doorbell);
1052                 ret = -EBUSY;
1053         }
1054 out:
1055         spin_unlock_irqrestore(hba->host->host_lock, flags);
1056         ufshcd_release(hba);
1057         return ret;
1058 }
1059
1060 /**
1061  * ufshcd_scale_gear - scale up/down UFS gear
1062  * @hba: per adapter instance
1063  * @scale_up: True for scaling up gear and false for scaling down
1064  *
1065  * Returns 0 for success,
1066  * Returns -EBUSY if scaling can't happen at this time
1067  * Returns non-zero for any other errors
1068  */
1069 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1070 {
1071         #define UFS_MIN_GEAR_TO_SCALE_DOWN      UFS_HS_G1
1072         int ret = 0;
1073         struct ufs_pa_layer_attr new_pwr_info;
1074
1075         if (scale_up) {
1076                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1077                        sizeof(struct ufs_pa_layer_attr));
1078         } else {
1079                 memcpy(&new_pwr_info, &hba->pwr_info,
1080                        sizeof(struct ufs_pa_layer_attr));
1081
1082                 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1083                     || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1084                         /* save the current power mode */
1085                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1086                                 &hba->pwr_info,
1087                                 sizeof(struct ufs_pa_layer_attr));
1088
1089                         /* scale down gear */
1090                         new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1091                         new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1092                 }
1093         }
1094
1095         /* check if the power mode needs to be changed or not? */
1096         ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1097
1098         if (ret)
1099                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1100                         __func__, ret,
1101                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1102                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1103
1104         return ret;
1105 }
1106
1107 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1108 {
1109         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1110         int ret = 0;
1111         /*
1112          * make sure that there are no outstanding requests when
1113          * clock scaling is in progress
1114          */
1115         ufshcd_scsi_block_requests(hba);
1116         down_write(&hba->clk_scaling_lock);
1117         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1118                 ret = -EBUSY;
1119                 up_write(&hba->clk_scaling_lock);
1120                 ufshcd_scsi_unblock_requests(hba);
1121         }
1122
1123         return ret;
1124 }
1125
1126 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1127 {
1128         up_write(&hba->clk_scaling_lock);
1129         ufshcd_scsi_unblock_requests(hba);
1130 }
1131
1132 /**
1133  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1134  * @hba: per adapter instance
1135  * @scale_up: True for scaling up and false for scalin down
1136  *
1137  * Returns 0 for success,
1138  * Returns -EBUSY if scaling can't happen at this time
1139  * Returns non-zero for any other errors
1140  */
1141 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1142 {
1143         int ret = 0;
1144
1145         /* let's not get into low power until clock scaling is completed */
1146         ufshcd_hold(hba, false);
1147
1148         ret = ufshcd_clock_scaling_prepare(hba);
1149         if (ret)
1150                 return ret;
1151
1152         /* scale down the gear before scaling down clocks */
1153         if (!scale_up) {
1154                 ret = ufshcd_scale_gear(hba, false);
1155                 if (ret)
1156                         goto out;
1157         }
1158
1159         ret = ufshcd_scale_clks(hba, scale_up);
1160         if (ret) {
1161                 if (!scale_up)
1162                         ufshcd_scale_gear(hba, true);
1163                 goto out;
1164         }
1165
1166         /* scale up the gear after scaling up clocks */
1167         if (scale_up) {
1168                 ret = ufshcd_scale_gear(hba, true);
1169                 if (ret) {
1170                         ufshcd_scale_clks(hba, false);
1171                         goto out;
1172                 }
1173         }
1174
1175         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1176
1177 out:
1178         ufshcd_clock_scaling_unprepare(hba);
1179         ufshcd_release(hba);
1180         return ret;
1181 }
1182
1183 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1184 {
1185         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1186                                            clk_scaling.suspend_work);
1187         unsigned long irq_flags;
1188
1189         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1190         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1191                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1192                 return;
1193         }
1194         hba->clk_scaling.is_suspended = true;
1195         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1196
1197         __ufshcd_suspend_clkscaling(hba);
1198 }
1199
1200 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1201 {
1202         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1203                                            clk_scaling.resume_work);
1204         unsigned long irq_flags;
1205
1206         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1207         if (!hba->clk_scaling.is_suspended) {
1208                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1209                 return;
1210         }
1211         hba->clk_scaling.is_suspended = false;
1212         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1213
1214         devfreq_resume_device(hba->devfreq);
1215 }
1216
1217 static int ufshcd_devfreq_target(struct device *dev,
1218                                 unsigned long *freq, u32 flags)
1219 {
1220         int ret = 0;
1221         struct ufs_hba *hba = dev_get_drvdata(dev);
1222         ktime_t start;
1223         bool scale_up, sched_clk_scaling_suspend_work = false;
1224         struct list_head *clk_list = &hba->clk_list_head;
1225         struct ufs_clk_info *clki;
1226         unsigned long irq_flags;
1227
1228         if (!ufshcd_is_clkscaling_supported(hba))
1229                 return -EINVAL;
1230
1231         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1232         if (ufshcd_eh_in_progress(hba)) {
1233                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1234                 return 0;
1235         }
1236
1237         if (!hba->clk_scaling.active_reqs)
1238                 sched_clk_scaling_suspend_work = true;
1239
1240         if (list_empty(clk_list)) {
1241                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1242                 goto out;
1243         }
1244
1245         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1246         scale_up = (*freq == clki->max_freq) ? true : false;
1247         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1248                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1249                 ret = 0;
1250                 goto out; /* no state change required */
1251         }
1252         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1253
1254         start = ktime_get();
1255         ret = ufshcd_devfreq_scale(hba, scale_up);
1256
1257         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1258                 (scale_up ? "up" : "down"),
1259                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1260
1261 out:
1262         if (sched_clk_scaling_suspend_work)
1263                 queue_work(hba->clk_scaling.workq,
1264                            &hba->clk_scaling.suspend_work);
1265
1266         return ret;
1267 }
1268
1269
1270 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1271                 struct devfreq_dev_status *stat)
1272 {
1273         struct ufs_hba *hba = dev_get_drvdata(dev);
1274         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1275         unsigned long flags;
1276
1277         if (!ufshcd_is_clkscaling_supported(hba))
1278                 return -EINVAL;
1279
1280         memset(stat, 0, sizeof(*stat));
1281
1282         spin_lock_irqsave(hba->host->host_lock, flags);
1283         if (!scaling->window_start_t)
1284                 goto start_window;
1285
1286         if (scaling->is_busy_started)
1287                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1288                                         scaling->busy_start_t));
1289
1290         stat->total_time = jiffies_to_usecs((long)jiffies -
1291                                 (long)scaling->window_start_t);
1292         stat->busy_time = scaling->tot_busy_t;
1293 start_window:
1294         scaling->window_start_t = jiffies;
1295         scaling->tot_busy_t = 0;
1296
1297         if (hba->outstanding_reqs) {
1298                 scaling->busy_start_t = ktime_get();
1299                 scaling->is_busy_started = true;
1300         } else {
1301                 scaling->busy_start_t = 0;
1302                 scaling->is_busy_started = false;
1303         }
1304         spin_unlock_irqrestore(hba->host->host_lock, flags);
1305         return 0;
1306 }
1307
1308 static struct devfreq_dev_profile ufs_devfreq_profile = {
1309         .polling_ms     = 100,
1310         .target         = ufshcd_devfreq_target,
1311         .get_dev_status = ufshcd_devfreq_get_dev_status,
1312 };
1313
1314 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1315 {
1316         struct list_head *clk_list = &hba->clk_list_head;
1317         struct ufs_clk_info *clki;
1318         struct devfreq *devfreq;
1319         int ret;
1320
1321         /* Skip devfreq if we don't have any clocks in the list */
1322         if (list_empty(clk_list))
1323                 return 0;
1324
1325         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1326         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1327         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1328
1329         devfreq = devfreq_add_device(hba->dev,
1330                         &ufs_devfreq_profile,
1331                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1332                         NULL);
1333         if (IS_ERR(devfreq)) {
1334                 ret = PTR_ERR(devfreq);
1335                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1336
1337                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1338                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1339                 return ret;
1340         }
1341
1342         hba->devfreq = devfreq;
1343
1344         return 0;
1345 }
1346
1347 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1348 {
1349         struct list_head *clk_list = &hba->clk_list_head;
1350         struct ufs_clk_info *clki;
1351
1352         if (!hba->devfreq)
1353                 return;
1354
1355         devfreq_remove_device(hba->devfreq);
1356         hba->devfreq = NULL;
1357
1358         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1359         dev_pm_opp_remove(hba->dev, clki->min_freq);
1360         dev_pm_opp_remove(hba->dev, clki->max_freq);
1361 }
1362
1363 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1364 {
1365         unsigned long flags;
1366
1367         devfreq_suspend_device(hba->devfreq);
1368         spin_lock_irqsave(hba->host->host_lock, flags);
1369         hba->clk_scaling.window_start_t = 0;
1370         spin_unlock_irqrestore(hba->host->host_lock, flags);
1371 }
1372
1373 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1374 {
1375         unsigned long flags;
1376         bool suspend = false;
1377
1378         if (!ufshcd_is_clkscaling_supported(hba))
1379                 return;
1380
1381         spin_lock_irqsave(hba->host->host_lock, flags);
1382         if (!hba->clk_scaling.is_suspended) {
1383                 suspend = true;
1384                 hba->clk_scaling.is_suspended = true;
1385         }
1386         spin_unlock_irqrestore(hba->host->host_lock, flags);
1387
1388         if (suspend)
1389                 __ufshcd_suspend_clkscaling(hba);
1390 }
1391
1392 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1393 {
1394         unsigned long flags;
1395         bool resume = false;
1396
1397         if (!ufshcd_is_clkscaling_supported(hba))
1398                 return;
1399
1400         spin_lock_irqsave(hba->host->host_lock, flags);
1401         if (hba->clk_scaling.is_suspended) {
1402                 resume = true;
1403                 hba->clk_scaling.is_suspended = false;
1404         }
1405         spin_unlock_irqrestore(hba->host->host_lock, flags);
1406
1407         if (resume)
1408                 devfreq_resume_device(hba->devfreq);
1409 }
1410
1411 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1412                 struct device_attribute *attr, char *buf)
1413 {
1414         struct ufs_hba *hba = dev_get_drvdata(dev);
1415
1416         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1417 }
1418
1419 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1420                 struct device_attribute *attr, const char *buf, size_t count)
1421 {
1422         struct ufs_hba *hba = dev_get_drvdata(dev);
1423         u32 value;
1424         int err;
1425
1426         if (kstrtou32(buf, 0, &value))
1427                 return -EINVAL;
1428
1429         value = !!value;
1430         if (value == hba->clk_scaling.is_allowed)
1431                 goto out;
1432
1433         pm_runtime_get_sync(hba->dev);
1434         ufshcd_hold(hba, false);
1435
1436         cancel_work_sync(&hba->clk_scaling.suspend_work);
1437         cancel_work_sync(&hba->clk_scaling.resume_work);
1438
1439         hba->clk_scaling.is_allowed = value;
1440
1441         if (value) {
1442                 ufshcd_resume_clkscaling(hba);
1443         } else {
1444                 ufshcd_suspend_clkscaling(hba);
1445                 err = ufshcd_devfreq_scale(hba, true);
1446                 if (err)
1447                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1448                                         __func__, err);
1449         }
1450
1451         ufshcd_release(hba);
1452         pm_runtime_put_sync(hba->dev);
1453 out:
1454         return count;
1455 }
1456
1457 static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1458 {
1459         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1460         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1461         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1462         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1463         hba->clk_scaling.enable_attr.attr.mode = 0644;
1464         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1465                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1466 }
1467
1468 static void ufshcd_ungate_work(struct work_struct *work)
1469 {
1470         int ret;
1471         unsigned long flags;
1472         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1473                         clk_gating.ungate_work);
1474
1475         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1476
1477         spin_lock_irqsave(hba->host->host_lock, flags);
1478         if (hba->clk_gating.state == CLKS_ON) {
1479                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1480                 goto unblock_reqs;
1481         }
1482
1483         spin_unlock_irqrestore(hba->host->host_lock, flags);
1484         ufshcd_setup_clocks(hba, true);
1485
1486         /* Exit from hibern8 */
1487         if (ufshcd_can_hibern8_during_gating(hba)) {
1488                 /* Prevent gating in this path */
1489                 hba->clk_gating.is_suspended = true;
1490                 if (ufshcd_is_link_hibern8(hba)) {
1491                         ret = ufshcd_uic_hibern8_exit(hba);
1492                         if (ret)
1493                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1494                                         __func__, ret);
1495                         else
1496                                 ufshcd_set_link_active(hba);
1497                 }
1498                 hba->clk_gating.is_suspended = false;
1499         }
1500 unblock_reqs:
1501         ufshcd_scsi_unblock_requests(hba);
1502 }
1503
1504 /**
1505  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1506  * Also, exit from hibern8 mode and set the link as active.
1507  * @hba: per adapter instance
1508  * @async: This indicates whether caller should ungate clocks asynchronously.
1509  */
1510 int ufshcd_hold(struct ufs_hba *hba, bool async)
1511 {
1512         int rc = 0;
1513         unsigned long flags;
1514
1515         if (!ufshcd_is_clkgating_allowed(hba))
1516                 goto out;
1517         spin_lock_irqsave(hba->host->host_lock, flags);
1518         hba->clk_gating.active_reqs++;
1519
1520         if (ufshcd_eh_in_progress(hba)) {
1521                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1522                 return 0;
1523         }
1524
1525 start:
1526         switch (hba->clk_gating.state) {
1527         case CLKS_ON:
1528                 /*
1529                  * Wait for the ungate work to complete if in progress.
1530                  * Though the clocks may be in ON state, the link could
1531                  * still be in hibner8 state if hibern8 is allowed
1532                  * during clock gating.
1533                  * Make sure we exit hibern8 state also in addition to
1534                  * clocks being ON.
1535                  */
1536                 if (ufshcd_can_hibern8_during_gating(hba) &&
1537                     ufshcd_is_link_hibern8(hba)) {
1538                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1539                         flush_work(&hba->clk_gating.ungate_work);
1540                         spin_lock_irqsave(hba->host->host_lock, flags);
1541                         goto start;
1542                 }
1543                 break;
1544         case REQ_CLKS_OFF:
1545                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1546                         hba->clk_gating.state = CLKS_ON;
1547                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1548                                                 hba->clk_gating.state);
1549                         break;
1550                 }
1551                 /*
1552                  * If we are here, it means gating work is either done or
1553                  * currently running. Hence, fall through to cancel gating
1554                  * work and to enable clocks.
1555                  */
1556                 /* fallthrough */
1557         case CLKS_OFF:
1558                 ufshcd_scsi_block_requests(hba);
1559                 hba->clk_gating.state = REQ_CLKS_ON;
1560                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1561                                         hba->clk_gating.state);
1562                 queue_work(hba->clk_gating.clk_gating_workq,
1563                            &hba->clk_gating.ungate_work);
1564                 /*
1565                  * fall through to check if we should wait for this
1566                  * work to be done or not.
1567                  */
1568                 /* fallthrough */
1569         case REQ_CLKS_ON:
1570                 if (async) {
1571                         rc = -EAGAIN;
1572                         hba->clk_gating.active_reqs--;
1573                         break;
1574                 }
1575
1576                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1577                 flush_work(&hba->clk_gating.ungate_work);
1578                 /* Make sure state is CLKS_ON before returning */
1579                 spin_lock_irqsave(hba->host->host_lock, flags);
1580                 goto start;
1581         default:
1582                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1583                                 __func__, hba->clk_gating.state);
1584                 break;
1585         }
1586         spin_unlock_irqrestore(hba->host->host_lock, flags);
1587 out:
1588         return rc;
1589 }
1590 EXPORT_SYMBOL_GPL(ufshcd_hold);
1591
1592 static void ufshcd_gate_work(struct work_struct *work)
1593 {
1594         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1595                         clk_gating.gate_work.work);
1596         unsigned long flags;
1597
1598         spin_lock_irqsave(hba->host->host_lock, flags);
1599         /*
1600          * In case you are here to cancel this work the gating state
1601          * would be marked as REQ_CLKS_ON. In this case save time by
1602          * skipping the gating work and exit after changing the clock
1603          * state to CLKS_ON.
1604          */
1605         if (hba->clk_gating.is_suspended ||
1606                 (hba->clk_gating.state == REQ_CLKS_ON)) {
1607                 hba->clk_gating.state = CLKS_ON;
1608                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1609                                         hba->clk_gating.state);
1610                 goto rel_lock;
1611         }
1612
1613         if (hba->clk_gating.active_reqs
1614                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1615                 || hba->lrb_in_use || hba->outstanding_tasks
1616                 || hba->active_uic_cmd || hba->uic_async_done)
1617                 goto rel_lock;
1618
1619         spin_unlock_irqrestore(hba->host->host_lock, flags);
1620
1621         /* put the link into hibern8 mode before turning off clocks */
1622         if (ufshcd_can_hibern8_during_gating(hba)) {
1623                 if (ufshcd_uic_hibern8_enter(hba)) {
1624                         hba->clk_gating.state = CLKS_ON;
1625                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1626                                                 hba->clk_gating.state);
1627                         goto out;
1628                 }
1629                 ufshcd_set_link_hibern8(hba);
1630         }
1631
1632         if (!ufshcd_is_link_active(hba))
1633                 ufshcd_setup_clocks(hba, false);
1634         else
1635                 /* If link is active, device ref_clk can't be switched off */
1636                 __ufshcd_setup_clocks(hba, false, true);
1637
1638         /*
1639          * In case you are here to cancel this work the gating state
1640          * would be marked as REQ_CLKS_ON. In this case keep the state
1641          * as REQ_CLKS_ON which would anyway imply that clocks are off
1642          * and a request to turn them on is pending. By doing this way,
1643          * we keep the state machine in tact and this would ultimately
1644          * prevent from doing cancel work multiple times when there are
1645          * new requests arriving before the current cancel work is done.
1646          */
1647         spin_lock_irqsave(hba->host->host_lock, flags);
1648         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1649                 hba->clk_gating.state = CLKS_OFF;
1650                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1651                                         hba->clk_gating.state);
1652         }
1653 rel_lock:
1654         spin_unlock_irqrestore(hba->host->host_lock, flags);
1655 out:
1656         return;
1657 }
1658
1659 /* host lock must be held before calling this variant */
1660 static void __ufshcd_release(struct ufs_hba *hba)
1661 {
1662         if (!ufshcd_is_clkgating_allowed(hba))
1663                 return;
1664
1665         hba->clk_gating.active_reqs--;
1666
1667         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1668                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1669                 || hba->lrb_in_use || hba->outstanding_tasks
1670                 || hba->active_uic_cmd || hba->uic_async_done
1671                 || ufshcd_eh_in_progress(hba))
1672                 return;
1673
1674         hba->clk_gating.state = REQ_CLKS_OFF;
1675         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1676         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1677                            &hba->clk_gating.gate_work,
1678                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1679 }
1680
1681 void ufshcd_release(struct ufs_hba *hba)
1682 {
1683         unsigned long flags;
1684
1685         spin_lock_irqsave(hba->host->host_lock, flags);
1686         __ufshcd_release(hba);
1687         spin_unlock_irqrestore(hba->host->host_lock, flags);
1688 }
1689 EXPORT_SYMBOL_GPL(ufshcd_release);
1690
1691 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1692                 struct device_attribute *attr, char *buf)
1693 {
1694         struct ufs_hba *hba = dev_get_drvdata(dev);
1695
1696         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1697 }
1698
1699 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1700                 struct device_attribute *attr, const char *buf, size_t count)
1701 {
1702         struct ufs_hba *hba = dev_get_drvdata(dev);
1703         unsigned long flags, value;
1704
1705         if (kstrtoul(buf, 0, &value))
1706                 return -EINVAL;
1707
1708         spin_lock_irqsave(hba->host->host_lock, flags);
1709         hba->clk_gating.delay_ms = value;
1710         spin_unlock_irqrestore(hba->host->host_lock, flags);
1711         return count;
1712 }
1713
1714 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1715                 struct device_attribute *attr, char *buf)
1716 {
1717         struct ufs_hba *hba = dev_get_drvdata(dev);
1718
1719         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1720 }
1721
1722 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1723                 struct device_attribute *attr, const char *buf, size_t count)
1724 {
1725         struct ufs_hba *hba = dev_get_drvdata(dev);
1726         unsigned long flags;
1727         u32 value;
1728
1729         if (kstrtou32(buf, 0, &value))
1730                 return -EINVAL;
1731
1732         value = !!value;
1733         if (value == hba->clk_gating.is_enabled)
1734                 goto out;
1735
1736         if (value) {
1737                 ufshcd_release(hba);
1738         } else {
1739                 spin_lock_irqsave(hba->host->host_lock, flags);
1740                 hba->clk_gating.active_reqs++;
1741                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1742         }
1743
1744         hba->clk_gating.is_enabled = value;
1745 out:
1746         return count;
1747 }
1748
1749 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1750 {
1751         char wq_name[sizeof("ufs_clkscaling_00")];
1752
1753         if (!ufshcd_is_clkscaling_supported(hba))
1754                 return;
1755
1756         INIT_WORK(&hba->clk_scaling.suspend_work,
1757                   ufshcd_clk_scaling_suspend_work);
1758         INIT_WORK(&hba->clk_scaling.resume_work,
1759                   ufshcd_clk_scaling_resume_work);
1760
1761         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1762                  hba->host->host_no);
1763         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1764
1765         ufshcd_clkscaling_init_sysfs(hba);
1766 }
1767
1768 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1769 {
1770         if (!ufshcd_is_clkscaling_supported(hba))
1771                 return;
1772
1773         destroy_workqueue(hba->clk_scaling.workq);
1774         ufshcd_devfreq_remove(hba);
1775 }
1776
1777 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1778 {
1779         char wq_name[sizeof("ufs_clk_gating_00")];
1780
1781         if (!ufshcd_is_clkgating_allowed(hba))
1782                 return;
1783
1784         hba->clk_gating.delay_ms = 150;
1785         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1786         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1787
1788         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1789                  hba->host->host_no);
1790         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1791                                                            WQ_MEM_RECLAIM);
1792
1793         hba->clk_gating.is_enabled = true;
1794
1795         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1796         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1797         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1798         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1799         hba->clk_gating.delay_attr.attr.mode = 0644;
1800         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1801                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1802
1803         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1804         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1805         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1806         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1807         hba->clk_gating.enable_attr.attr.mode = 0644;
1808         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1809                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1810 }
1811
1812 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1813 {
1814         if (!ufshcd_is_clkgating_allowed(hba))
1815                 return;
1816         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1817         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1818         cancel_work_sync(&hba->clk_gating.ungate_work);
1819         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1820         destroy_workqueue(hba->clk_gating.clk_gating_workq);
1821 }
1822
1823 /* Must be called with host lock acquired */
1824 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1825 {
1826         bool queue_resume_work = false;
1827
1828         if (!ufshcd_is_clkscaling_supported(hba))
1829                 return;
1830
1831         if (!hba->clk_scaling.active_reqs++)
1832                 queue_resume_work = true;
1833
1834         if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1835                 return;
1836
1837         if (queue_resume_work)
1838                 queue_work(hba->clk_scaling.workq,
1839                            &hba->clk_scaling.resume_work);
1840
1841         if (!hba->clk_scaling.window_start_t) {
1842                 hba->clk_scaling.window_start_t = jiffies;
1843                 hba->clk_scaling.tot_busy_t = 0;
1844                 hba->clk_scaling.is_busy_started = false;
1845         }
1846
1847         if (!hba->clk_scaling.is_busy_started) {
1848                 hba->clk_scaling.busy_start_t = ktime_get();
1849                 hba->clk_scaling.is_busy_started = true;
1850         }
1851 }
1852
1853 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1854 {
1855         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1856
1857         if (!ufshcd_is_clkscaling_supported(hba))
1858                 return;
1859
1860         if (!hba->outstanding_reqs && scaling->is_busy_started) {
1861                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1862                                         scaling->busy_start_t));
1863                 scaling->busy_start_t = 0;
1864                 scaling->is_busy_started = false;
1865         }
1866 }
1867 /**
1868  * ufshcd_send_command - Send SCSI or device management commands
1869  * @hba: per adapter instance
1870  * @task_tag: Task tag of the command
1871  */
1872 static inline
1873 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1874 {
1875         hba->lrb[task_tag].issue_time_stamp = ktime_get();
1876         hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
1877         ufshcd_clk_scaling_start_busy(hba);
1878         __set_bit(task_tag, &hba->outstanding_reqs);
1879         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1880         /* Make sure that doorbell is committed immediately */
1881         wmb();
1882         ufshcd_add_command_trace(hba, task_tag, "send");
1883 }
1884
1885 /**
1886  * ufshcd_copy_sense_data - Copy sense data in case of check condition
1887  * @lrbp: pointer to local reference block
1888  */
1889 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1890 {
1891         int len;
1892         if (lrbp->sense_buffer &&
1893             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1894                 int len_to_copy;
1895
1896                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1897                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
1898
1899                 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1900                        len_to_copy);
1901         }
1902 }
1903
1904 /**
1905  * ufshcd_copy_query_response() - Copy the Query Response and the data
1906  * descriptor
1907  * @hba: per adapter instance
1908  * @lrbp: pointer to local reference block
1909  */
1910 static
1911 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1912 {
1913         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1914
1915         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
1916
1917         /* Get the descriptor */
1918         if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
1919                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
1920                                 GENERAL_UPIU_REQUEST_SIZE;
1921                 u16 resp_len;
1922                 u16 buf_len;
1923
1924                 /* data segment length */
1925                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
1926                                                 MASK_QUERY_DATA_SEG_LEN;
1927                 buf_len = be16_to_cpu(
1928                                 hba->dev_cmd.query.request.upiu_req.length);
1929                 if (likely(buf_len >= resp_len)) {
1930                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1931                 } else {
1932                         dev_warn(hba->dev,
1933                                 "%s: Response size is bigger than buffer",
1934                                 __func__);
1935                         return -EINVAL;
1936                 }
1937         }
1938
1939         return 0;
1940 }
1941
1942 /**
1943  * ufshcd_hba_capabilities - Read controller capabilities
1944  * @hba: per adapter instance
1945  */
1946 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1947 {
1948         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1949
1950         /* nutrs and nutmrs are 0 based values */
1951         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1952         hba->nutmrs =
1953         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1954 }
1955
1956 /**
1957  * ufshcd_ready_for_uic_cmd - Check if controller is ready
1958  *                            to accept UIC commands
1959  * @hba: per adapter instance
1960  * Return true on success, else false
1961  */
1962 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1963 {
1964         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1965                 return true;
1966         else
1967                 return false;
1968 }
1969
1970 /**
1971  * ufshcd_get_upmcrs - Get the power mode change request status
1972  * @hba: Pointer to adapter instance
1973  *
1974  * This function gets the UPMCRS field of HCS register
1975  * Returns value of UPMCRS field
1976  */
1977 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1978 {
1979         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1980 }
1981
1982 /**
1983  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1984  * @hba: per adapter instance
1985  * @uic_cmd: UIC command
1986  *
1987  * Mutex must be held.
1988  */
1989 static inline void
1990 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1991 {
1992         WARN_ON(hba->active_uic_cmd);
1993
1994         hba->active_uic_cmd = uic_cmd;
1995
1996         /* Write Args */
1997         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1998         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1999         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2000
2001         /* Write UIC Cmd */
2002         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2003                       REG_UIC_COMMAND);
2004 }
2005
2006 /**
2007  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2008  * @hba: per adapter instance
2009  * @uic_cmd: UIC command
2010  *
2011  * Must be called with mutex held.
2012  * Returns 0 only if success.
2013  */
2014 static int
2015 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2016 {
2017         int ret;
2018         unsigned long flags;
2019
2020         if (wait_for_completion_timeout(&uic_cmd->done,
2021                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2022                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2023         else
2024                 ret = -ETIMEDOUT;
2025
2026         spin_lock_irqsave(hba->host->host_lock, flags);
2027         hba->active_uic_cmd = NULL;
2028         spin_unlock_irqrestore(hba->host->host_lock, flags);
2029
2030         return ret;
2031 }
2032
2033 /**
2034  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2035  * @hba: per adapter instance
2036  * @uic_cmd: UIC command
2037  * @completion: initialize the completion only if this is set to true
2038  *
2039  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2040  * with mutex held and host_lock locked.
2041  * Returns 0 only if success.
2042  */
2043 static int
2044 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2045                       bool completion)
2046 {
2047         if (!ufshcd_ready_for_uic_cmd(hba)) {
2048                 dev_err(hba->dev,
2049                         "Controller not ready to accept UIC commands\n");
2050                 return -EIO;
2051         }
2052
2053         if (completion)
2054                 init_completion(&uic_cmd->done);
2055
2056         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2057
2058         return 0;
2059 }
2060
2061 /**
2062  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2063  * @hba: per adapter instance
2064  * @uic_cmd: UIC command
2065  *
2066  * Returns 0 only if success.
2067  */
2068 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2069 {
2070         int ret;
2071         unsigned long flags;
2072
2073         ufshcd_hold(hba, false);
2074         mutex_lock(&hba->uic_cmd_mutex);
2075         ufshcd_add_delay_before_dme_cmd(hba);
2076
2077         spin_lock_irqsave(hba->host->host_lock, flags);
2078         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2079         spin_unlock_irqrestore(hba->host->host_lock, flags);
2080         if (!ret)
2081                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2082
2083         mutex_unlock(&hba->uic_cmd_mutex);
2084
2085         ufshcd_release(hba);
2086         return ret;
2087 }
2088
2089 /**
2090  * ufshcd_map_sg - Map scatter-gather list to prdt
2091  * @hba: per adapter instance
2092  * @lrbp: pointer to local reference block
2093  *
2094  * Returns 0 in case of success, non-zero value in case of failure
2095  */
2096 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2097 {
2098         struct ufshcd_sg_entry *prd_table;
2099         struct scatterlist *sg;
2100         struct scsi_cmnd *cmd;
2101         int sg_segments;
2102         int i;
2103
2104         cmd = lrbp->cmd;
2105         sg_segments = scsi_dma_map(cmd);
2106         if (sg_segments < 0)
2107                 return sg_segments;
2108
2109         if (sg_segments) {
2110                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2111                         lrbp->utr_descriptor_ptr->prd_table_length =
2112                                 cpu_to_le16((u16)(sg_segments *
2113                                         sizeof(struct ufshcd_sg_entry)));
2114                 else
2115                         lrbp->utr_descriptor_ptr->prd_table_length =
2116                                 cpu_to_le16((u16) (sg_segments));
2117
2118                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2119
2120                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2121                         prd_table[i].size  =
2122                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2123                         prd_table[i].base_addr =
2124                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2125                         prd_table[i].upper_addr =
2126                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2127                         prd_table[i].reserved = 0;
2128                 }
2129         } else {
2130                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2131         }
2132
2133         return 0;
2134 }
2135
2136 /**
2137  * ufshcd_enable_intr - enable interrupts
2138  * @hba: per adapter instance
2139  * @intrs: interrupt bits
2140  */
2141 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2142 {
2143         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2144
2145         if (hba->ufs_version == UFSHCI_VERSION_10) {
2146                 u32 rw;
2147                 rw = set & INTERRUPT_MASK_RW_VER_10;
2148                 set = rw | ((set ^ intrs) & intrs);
2149         } else {
2150                 set |= intrs;
2151         }
2152
2153         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2154 }
2155
2156 /**
2157  * ufshcd_disable_intr - disable interrupts
2158  * @hba: per adapter instance
2159  * @intrs: interrupt bits
2160  */
2161 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2162 {
2163         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2164
2165         if (hba->ufs_version == UFSHCI_VERSION_10) {
2166                 u32 rw;
2167                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2168                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2169                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2170
2171         } else {
2172                 set &= ~intrs;
2173         }
2174
2175         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2176 }
2177
2178 /**
2179  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2180  * descriptor according to request
2181  * @lrbp: pointer to local reference block
2182  * @upiu_flags: flags required in the header
2183  * @cmd_dir: requests data direction
2184  */
2185 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2186                         u32 *upiu_flags, enum dma_data_direction cmd_dir)
2187 {
2188         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2189         u32 data_direction;
2190         u32 dword_0;
2191
2192         if (cmd_dir == DMA_FROM_DEVICE) {
2193                 data_direction = UTP_DEVICE_TO_HOST;
2194                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2195         } else if (cmd_dir == DMA_TO_DEVICE) {
2196                 data_direction = UTP_HOST_TO_DEVICE;
2197                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2198         } else {
2199                 data_direction = UTP_NO_DATA_TRANSFER;
2200                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2201         }
2202
2203         dword_0 = data_direction | (lrbp->command_type
2204                                 << UPIU_COMMAND_TYPE_OFFSET);
2205         if (lrbp->intr_cmd)
2206                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2207
2208         /* Transfer request descriptor header fields */
2209         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2210         /* dword_1 is reserved, hence it is set to 0 */
2211         req_desc->header.dword_1 = 0;
2212         /*
2213          * assigning invalid value for command status. Controller
2214          * updates OCS on command completion, with the command
2215          * status
2216          */
2217         req_desc->header.dword_2 =
2218                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2219         /* dword_3 is reserved, hence it is set to 0 */
2220         req_desc->header.dword_3 = 0;
2221
2222         req_desc->prd_table_length = 0;
2223 }
2224
2225 /**
2226  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2227  * for scsi commands
2228  * @lrbp: local reference block pointer
2229  * @upiu_flags: flags
2230  */
2231 static
2232 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2233 {
2234         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2235         unsigned short cdb_len;
2236
2237         /* command descriptor fields */
2238         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2239                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2240                                 lrbp->lun, lrbp->task_tag);
2241         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2242                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2243
2244         /* Total EHS length and Data segment length will be zero */
2245         ucd_req_ptr->header.dword_2 = 0;
2246
2247         ucd_req_ptr->sc.exp_data_transfer_len =
2248                 cpu_to_be32(lrbp->cmd->sdb.length);
2249
2250         cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
2251         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2252         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2253
2254         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2255 }
2256
2257 /**
2258  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2259  * for query requsts
2260  * @hba: UFS hba
2261  * @lrbp: local reference block pointer
2262  * @upiu_flags: flags
2263  */
2264 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2265                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2266 {
2267         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2268         struct ufs_query *query = &hba->dev_cmd.query;
2269         u16 len = be16_to_cpu(query->request.upiu_req.length);
2270
2271         /* Query request header */
2272         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2273                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2274                         lrbp->lun, lrbp->task_tag);
2275         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2276                         0, query->request.query_func, 0, 0);
2277
2278         /* Data segment length only need for WRITE_DESC */
2279         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2280                 ucd_req_ptr->header.dword_2 =
2281                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2282         else
2283                 ucd_req_ptr->header.dword_2 = 0;
2284
2285         /* Copy the Query Request buffer as is */
2286         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2287                         QUERY_OSF_SIZE);
2288
2289         /* Copy the Descriptor */
2290         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2291                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2292
2293         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2294 }
2295
2296 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2297 {
2298         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2299
2300         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2301
2302         /* command descriptor fields */
2303         ucd_req_ptr->header.dword_0 =
2304                 UPIU_HEADER_DWORD(
2305                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2306         /* clear rest of the fields of basic header */
2307         ucd_req_ptr->header.dword_1 = 0;
2308         ucd_req_ptr->header.dword_2 = 0;
2309
2310         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2311 }
2312
2313 /**
2314  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2315  *                           for Device Management Purposes
2316  * @hba: per adapter instance
2317  * @lrbp: pointer to local reference block
2318  */
2319 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2320 {
2321         u32 upiu_flags;
2322         int ret = 0;
2323
2324         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2325             (hba->ufs_version == UFSHCI_VERSION_11))
2326                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2327         else
2328                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2329
2330         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2331         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2332                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2333         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2334                 ufshcd_prepare_utp_nop_upiu(lrbp);
2335         else
2336                 ret = -EINVAL;
2337
2338         return ret;
2339 }
2340
2341 /**
2342  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2343  *                         for SCSI Purposes
2344  * @hba: per adapter instance
2345  * @lrbp: pointer to local reference block
2346  */
2347 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2348 {
2349         u32 upiu_flags;
2350         int ret = 0;
2351
2352         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2353             (hba->ufs_version == UFSHCI_VERSION_11))
2354                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2355         else
2356                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2357
2358         if (likely(lrbp->cmd)) {
2359                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2360                                                 lrbp->cmd->sc_data_direction);
2361                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2362         } else {
2363                 ret = -EINVAL;
2364         }
2365
2366         return ret;
2367 }
2368
2369 /**
2370  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2371  * @upiu_wlun_id: UPIU W-LUN id
2372  *
2373  * Returns SCSI W-LUN id
2374  */
2375 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2376 {
2377         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2378 }
2379
2380 /**
2381  * ufshcd_queuecommand - main entry point for SCSI requests
2382  * @host: SCSI host pointer
2383  * @cmd: command from SCSI Midlayer
2384  *
2385  * Returns 0 for success, non-zero in case of failure
2386  */
2387 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2388 {
2389         struct ufshcd_lrb *lrbp;
2390         struct ufs_hba *hba;
2391         unsigned long flags;
2392         int tag;
2393         int err = 0;
2394
2395         hba = shost_priv(host);
2396
2397         tag = cmd->request->tag;
2398         if (!ufshcd_valid_tag(hba, tag)) {
2399                 dev_err(hba->dev,
2400                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2401                         __func__, tag, cmd, cmd->request);
2402                 BUG();
2403         }
2404
2405         if (!down_read_trylock(&hba->clk_scaling_lock))
2406                 return SCSI_MLQUEUE_HOST_BUSY;
2407
2408         spin_lock_irqsave(hba->host->host_lock, flags);
2409         switch (hba->ufshcd_state) {
2410         case UFSHCD_STATE_OPERATIONAL:
2411                 break;
2412         case UFSHCD_STATE_EH_SCHEDULED:
2413         case UFSHCD_STATE_RESET:
2414                 err = SCSI_MLQUEUE_HOST_BUSY;
2415                 goto out_unlock;
2416         case UFSHCD_STATE_ERROR:
2417                 set_host_byte(cmd, DID_ERROR);
2418                 cmd->scsi_done(cmd);
2419                 goto out_unlock;
2420         default:
2421                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2422                                 __func__, hba->ufshcd_state);
2423                 set_host_byte(cmd, DID_BAD_TARGET);
2424                 cmd->scsi_done(cmd);
2425                 goto out_unlock;
2426         }
2427
2428         /* if error handling is in progress, don't issue commands */
2429         if (ufshcd_eh_in_progress(hba)) {
2430                 set_host_byte(cmd, DID_ERROR);
2431                 cmd->scsi_done(cmd);
2432                 goto out_unlock;
2433         }
2434         spin_unlock_irqrestore(hba->host->host_lock, flags);
2435
2436         hba->req_abort_count = 0;
2437
2438         /* acquire the tag to make sure device cmds don't use it */
2439         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
2440                 /*
2441                  * Dev manage command in progress, requeue the command.
2442                  * Requeuing the command helps in cases where the request *may*
2443                  * find different tag instead of waiting for dev manage command
2444                  * completion.
2445                  */
2446                 err = SCSI_MLQUEUE_HOST_BUSY;
2447                 goto out;
2448         }
2449
2450         err = ufshcd_hold(hba, true);
2451         if (err) {
2452                 err = SCSI_MLQUEUE_HOST_BUSY;
2453                 clear_bit_unlock(tag, &hba->lrb_in_use);
2454                 goto out;
2455         }
2456         WARN_ON(hba->clk_gating.state != CLKS_ON);
2457
2458         lrbp = &hba->lrb[tag];
2459
2460         WARN_ON(lrbp->cmd);
2461         lrbp->cmd = cmd;
2462         lrbp->sense_bufflen = UFS_SENSE_SIZE;
2463         lrbp->sense_buffer = cmd->sense_buffer;
2464         lrbp->task_tag = tag;
2465         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2466         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2467         lrbp->req_abort_skip = false;
2468
2469         ufshcd_comp_scsi_upiu(hba, lrbp);
2470
2471         err = ufshcd_map_sg(hba, lrbp);
2472         if (err) {
2473                 lrbp->cmd = NULL;
2474                 clear_bit_unlock(tag, &hba->lrb_in_use);
2475                 goto out;
2476         }
2477         /* Make sure descriptors are ready before ringing the doorbell */
2478         wmb();
2479
2480         /* issue command to the controller */
2481         spin_lock_irqsave(hba->host->host_lock, flags);
2482         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2483         ufshcd_send_command(hba, tag);
2484 out_unlock:
2485         spin_unlock_irqrestore(hba->host->host_lock, flags);
2486 out:
2487         up_read(&hba->clk_scaling_lock);
2488         return err;
2489 }
2490
2491 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2492                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2493 {
2494         lrbp->cmd = NULL;
2495         lrbp->sense_bufflen = 0;
2496         lrbp->sense_buffer = NULL;
2497         lrbp->task_tag = tag;
2498         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2499         lrbp->intr_cmd = true; /* No interrupt aggregation */
2500         hba->dev_cmd.type = cmd_type;
2501
2502         return ufshcd_comp_devman_upiu(hba, lrbp);
2503 }
2504
2505 static int
2506 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2507 {
2508         int err = 0;
2509         unsigned long flags;
2510         u32 mask = 1 << tag;
2511
2512         /* clear outstanding transaction before retry */
2513         spin_lock_irqsave(hba->host->host_lock, flags);
2514         ufshcd_utrl_clear(hba, tag);
2515         spin_unlock_irqrestore(hba->host->host_lock, flags);
2516
2517         /*
2518          * wait for for h/w to clear corresponding bit in door-bell.
2519          * max. wait is 1 sec.
2520          */
2521         err = ufshcd_wait_for_register(hba,
2522                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2523                         mask, ~mask, 1000, 1000, true);
2524
2525         return err;
2526 }
2527
2528 static int
2529 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2530 {
2531         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2532
2533         /* Get the UPIU response */
2534         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2535                                 UPIU_RSP_CODE_OFFSET;
2536         return query_res->response;
2537 }
2538
2539 /**
2540  * ufshcd_dev_cmd_completion() - handles device management command responses
2541  * @hba: per adapter instance
2542  * @lrbp: pointer to local reference block
2543  */
2544 static int
2545 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2546 {
2547         int resp;
2548         int err = 0;
2549
2550         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2551         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2552
2553         switch (resp) {
2554         case UPIU_TRANSACTION_NOP_IN:
2555                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2556                         err = -EINVAL;
2557                         dev_err(hba->dev, "%s: unexpected response %x\n",
2558                                         __func__, resp);
2559                 }
2560                 break;
2561         case UPIU_TRANSACTION_QUERY_RSP:
2562                 err = ufshcd_check_query_response(hba, lrbp);
2563                 if (!err)
2564                         err = ufshcd_copy_query_response(hba, lrbp);
2565                 break;
2566         case UPIU_TRANSACTION_REJECT_UPIU:
2567                 /* TODO: handle Reject UPIU Response */
2568                 err = -EPERM;
2569                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2570                                 __func__);
2571                 break;
2572         default:
2573                 err = -EINVAL;
2574                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2575                                 __func__, resp);
2576                 break;
2577         }
2578
2579         return err;
2580 }
2581
2582 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2583                 struct ufshcd_lrb *lrbp, int max_timeout)
2584 {
2585         int err = 0;
2586         unsigned long time_left;
2587         unsigned long flags;
2588
2589         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2590                         msecs_to_jiffies(max_timeout));
2591
2592         /* Make sure descriptors are ready before ringing the doorbell */
2593         wmb();
2594         spin_lock_irqsave(hba->host->host_lock, flags);
2595         hba->dev_cmd.complete = NULL;
2596         if (likely(time_left)) {
2597                 err = ufshcd_get_tr_ocs(lrbp);
2598                 if (!err)
2599                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2600         }
2601         spin_unlock_irqrestore(hba->host->host_lock, flags);
2602
2603         if (!time_left) {
2604                 err = -ETIMEDOUT;
2605                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2606                         __func__, lrbp->task_tag);
2607                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2608                         /* successfully cleared the command, retry if needed */
2609                         err = -EAGAIN;
2610                 /*
2611                  * in case of an error, after clearing the doorbell,
2612                  * we also need to clear the outstanding_request
2613                  * field in hba
2614                  */
2615                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2616         }
2617
2618         return err;
2619 }
2620
2621 /**
2622  * ufshcd_get_dev_cmd_tag - Get device management command tag
2623  * @hba: per-adapter instance
2624  * @tag_out: pointer to variable with available slot value
2625  *
2626  * Get a free slot and lock it until device management command
2627  * completes.
2628  *
2629  * Returns false if free slot is unavailable for locking, else
2630  * return true with tag value in @tag.
2631  */
2632 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
2633 {
2634         int tag;
2635         bool ret = false;
2636         unsigned long tmp;
2637
2638         if (!tag_out)
2639                 goto out;
2640
2641         do {
2642                 tmp = ~hba->lrb_in_use;
2643                 tag = find_last_bit(&tmp, hba->nutrs);
2644                 if (tag >= hba->nutrs)
2645                         goto out;
2646         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
2647
2648         *tag_out = tag;
2649         ret = true;
2650 out:
2651         return ret;
2652 }
2653
2654 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
2655 {
2656         clear_bit_unlock(tag, &hba->lrb_in_use);
2657 }
2658
2659 /**
2660  * ufshcd_exec_dev_cmd - API for sending device management requests
2661  * @hba: UFS hba
2662  * @cmd_type: specifies the type (NOP, Query...)
2663  * @timeout: time in seconds
2664  *
2665  * NOTE: Since there is only one available tag for device management commands,
2666  * it is expected you hold the hba->dev_cmd.lock mutex.
2667  */
2668 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2669                 enum dev_cmd_type cmd_type, int timeout)
2670 {
2671         struct ufshcd_lrb *lrbp;
2672         int err;
2673         int tag;
2674         struct completion wait;
2675         unsigned long flags;
2676
2677         down_read(&hba->clk_scaling_lock);
2678
2679         /*
2680          * Get free slot, sleep if slots are unavailable.
2681          * Even though we use wait_event() which sleeps indefinitely,
2682          * the maximum wait time is bounded by SCSI request timeout.
2683          */
2684         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
2685
2686         init_completion(&wait);
2687         lrbp = &hba->lrb[tag];
2688         WARN_ON(lrbp->cmd);
2689         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2690         if (unlikely(err))
2691                 goto out_put_tag;
2692
2693         hba->dev_cmd.complete = &wait;
2694
2695         ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2696         /* Make sure descriptors are ready before ringing the doorbell */
2697         wmb();
2698         spin_lock_irqsave(hba->host->host_lock, flags);
2699         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2700         ufshcd_send_command(hba, tag);
2701         spin_unlock_irqrestore(hba->host->host_lock, flags);
2702
2703         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2704
2705         ufshcd_add_query_upiu_trace(hba, tag,
2706                         err ? "query_complete_err" : "query_complete");
2707
2708 out_put_tag:
2709         ufshcd_put_dev_cmd_tag(hba, tag);
2710         wake_up(&hba->dev_cmd.tag_wq);
2711         up_read(&hba->clk_scaling_lock);
2712         return err;
2713 }
2714
2715 /**
2716  * ufshcd_init_query() - init the query response and request parameters
2717  * @hba: per-adapter instance
2718  * @request: address of the request pointer to be initialized
2719  * @response: address of the response pointer to be initialized
2720  * @opcode: operation to perform
2721  * @idn: flag idn to access
2722  * @index: LU number to access
2723  * @selector: query/flag/descriptor further identification
2724  */
2725 static inline void ufshcd_init_query(struct ufs_hba *hba,
2726                 struct ufs_query_req **request, struct ufs_query_res **response,
2727                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2728 {
2729         *request = &hba->dev_cmd.query.request;
2730         *response = &hba->dev_cmd.query.response;
2731         memset(*request, 0, sizeof(struct ufs_query_req));
2732         memset(*response, 0, sizeof(struct ufs_query_res));
2733         (*request)->upiu_req.opcode = opcode;
2734         (*request)->upiu_req.idn = idn;
2735         (*request)->upiu_req.index = index;
2736         (*request)->upiu_req.selector = selector;
2737 }
2738
2739 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2740         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2741 {
2742         int ret;
2743         int retries;
2744
2745         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2746                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2747                 if (ret)
2748                         dev_dbg(hba->dev,
2749                                 "%s: failed with error %d, retries %d\n",
2750                                 __func__, ret, retries);
2751                 else
2752                         break;
2753         }
2754
2755         if (ret)
2756                 dev_err(hba->dev,
2757                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2758                         __func__, opcode, idn, ret, retries);
2759         return ret;
2760 }
2761
2762 /**
2763  * ufshcd_query_flag() - API function for sending flag query requests
2764  * @hba: per-adapter instance
2765  * @opcode: flag query to perform
2766  * @idn: flag idn to access
2767  * @flag_res: the flag value after the query request completes
2768  *
2769  * Returns 0 for success, non-zero in case of failure
2770  */
2771 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2772                         enum flag_idn idn, bool *flag_res)
2773 {
2774         struct ufs_query_req *request = NULL;
2775         struct ufs_query_res *response = NULL;
2776         int err, index = 0, selector = 0;
2777         int timeout = QUERY_REQ_TIMEOUT;
2778
2779         BUG_ON(!hba);
2780
2781         ufshcd_hold(hba, false);
2782         mutex_lock(&hba->dev_cmd.lock);
2783         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2784                         selector);
2785
2786         switch (opcode) {
2787         case UPIU_QUERY_OPCODE_SET_FLAG:
2788         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2789         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2790                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2791                 break;
2792         case UPIU_QUERY_OPCODE_READ_FLAG:
2793                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2794                 if (!flag_res) {
2795                         /* No dummy reads */
2796                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
2797                                         __func__);
2798                         err = -EINVAL;
2799                         goto out_unlock;
2800                 }
2801                 break;
2802         default:
2803                 dev_err(hba->dev,
2804                         "%s: Expected query flag opcode but got = %d\n",
2805                         __func__, opcode);
2806                 err = -EINVAL;
2807                 goto out_unlock;
2808         }
2809
2810         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2811
2812         if (err) {
2813                 dev_err(hba->dev,
2814                         "%s: Sending flag query for idn %d failed, err = %d\n",
2815                         __func__, idn, err);
2816                 goto out_unlock;
2817         }
2818
2819         if (flag_res)
2820                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2821                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2822
2823 out_unlock:
2824         mutex_unlock(&hba->dev_cmd.lock);
2825         ufshcd_release(hba);
2826         return err;
2827 }
2828
2829 /**
2830  * ufshcd_query_attr - API function for sending attribute requests
2831  * @hba: per-adapter instance
2832  * @opcode: attribute opcode
2833  * @idn: attribute idn to access
2834  * @index: index field
2835  * @selector: selector field
2836  * @attr_val: the attribute value after the query request completes
2837  *
2838  * Returns 0 for success, non-zero in case of failure
2839 */
2840 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2841                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2842 {
2843         struct ufs_query_req *request = NULL;
2844         struct ufs_query_res *response = NULL;
2845         int err;
2846
2847         BUG_ON(!hba);
2848
2849         ufshcd_hold(hba, false);
2850         if (!attr_val) {
2851                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2852                                 __func__, opcode);
2853                 err = -EINVAL;
2854                 goto out;
2855         }
2856
2857         mutex_lock(&hba->dev_cmd.lock);
2858         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2859                         selector);
2860
2861         switch (opcode) {
2862         case UPIU_QUERY_OPCODE_WRITE_ATTR:
2863                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2864                 request->upiu_req.value = cpu_to_be32(*attr_val);
2865                 break;
2866         case UPIU_QUERY_OPCODE_READ_ATTR:
2867                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2868                 break;
2869         default:
2870                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2871                                 __func__, opcode);
2872                 err = -EINVAL;
2873                 goto out_unlock;
2874         }
2875
2876         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2877
2878         if (err) {
2879                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2880                                 __func__, opcode, idn, index, err);
2881                 goto out_unlock;
2882         }
2883
2884         *attr_val = be32_to_cpu(response->upiu_res.value);
2885
2886 out_unlock:
2887         mutex_unlock(&hba->dev_cmd.lock);
2888 out:
2889         ufshcd_release(hba);
2890         return err;
2891 }
2892
2893 /**
2894  * ufshcd_query_attr_retry() - API function for sending query
2895  * attribute with retries
2896  * @hba: per-adapter instance
2897  * @opcode: attribute opcode
2898  * @idn: attribute idn to access
2899  * @index: index field
2900  * @selector: selector field
2901  * @attr_val: the attribute value after the query request
2902  * completes
2903  *
2904  * Returns 0 for success, non-zero in case of failure
2905 */
2906 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2907         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2908         u32 *attr_val)
2909 {
2910         int ret = 0;
2911         u32 retries;
2912
2913          for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2914                 ret = ufshcd_query_attr(hba, opcode, idn, index,
2915                                                 selector, attr_val);
2916                 if (ret)
2917                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2918                                 __func__, ret, retries);
2919                 else
2920                         break;
2921         }
2922
2923         if (ret)
2924                 dev_err(hba->dev,
2925                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2926                         __func__, idn, ret, QUERY_REQ_RETRIES);
2927         return ret;
2928 }
2929
2930 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
2931                         enum query_opcode opcode, enum desc_idn idn, u8 index,
2932                         u8 selector, u8 *desc_buf, int *buf_len)
2933 {
2934         struct ufs_query_req *request = NULL;
2935         struct ufs_query_res *response = NULL;
2936         int err;
2937
2938         BUG_ON(!hba);
2939
2940         ufshcd_hold(hba, false);
2941         if (!desc_buf) {
2942                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2943                                 __func__, opcode);
2944                 err = -EINVAL;
2945                 goto out;
2946         }
2947
2948         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
2949                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2950                                 __func__, *buf_len);
2951                 err = -EINVAL;
2952                 goto out;
2953         }
2954
2955         mutex_lock(&hba->dev_cmd.lock);
2956         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2957                         selector);
2958         hba->dev_cmd.query.descriptor = desc_buf;
2959         request->upiu_req.length = cpu_to_be16(*buf_len);
2960
2961         switch (opcode) {
2962         case UPIU_QUERY_OPCODE_WRITE_DESC:
2963                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2964                 break;
2965         case UPIU_QUERY_OPCODE_READ_DESC:
2966                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2967                 break;
2968         default:
2969                 dev_err(hba->dev,
2970                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2971                                 __func__, opcode);
2972                 err = -EINVAL;
2973                 goto out_unlock;
2974         }
2975
2976         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2977
2978         if (err) {
2979                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2980                                 __func__, opcode, idn, index, err);
2981                 goto out_unlock;
2982         }
2983
2984         hba->dev_cmd.query.descriptor = NULL;
2985         *buf_len = be16_to_cpu(response->upiu_res.length);
2986
2987 out_unlock:
2988         mutex_unlock(&hba->dev_cmd.lock);
2989 out:
2990         ufshcd_release(hba);
2991         return err;
2992 }
2993
2994 /**
2995  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2996  * @hba: per-adapter instance
2997  * @opcode: attribute opcode
2998  * @idn: attribute idn to access
2999  * @index: index field
3000  * @selector: selector field
3001  * @desc_buf: the buffer that contains the descriptor
3002  * @buf_len: length parameter passed to the device
3003  *
3004  * Returns 0 for success, non-zero in case of failure.
3005  * The buf_len parameter will contain, on return, the length parameter
3006  * received on the response.
3007  */
3008 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3009                                   enum query_opcode opcode,
3010                                   enum desc_idn idn, u8 index,
3011                                   u8 selector,
3012                                   u8 *desc_buf, int *buf_len)
3013 {
3014         int err;
3015         int retries;
3016
3017         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3018                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3019                                                 selector, desc_buf, buf_len);
3020                 if (!err || err == -EINVAL)
3021                         break;
3022         }
3023
3024         return err;
3025 }
3026
3027 /**
3028  * ufshcd_read_desc_length - read the specified descriptor length from header
3029  * @hba: Pointer to adapter instance
3030  * @desc_id: descriptor idn value
3031  * @desc_index: descriptor index
3032  * @desc_length: pointer to variable to read the length of descriptor
3033  *
3034  * Return 0 in case of success, non-zero otherwise
3035  */
3036 static int ufshcd_read_desc_length(struct ufs_hba *hba,
3037         enum desc_idn desc_id,
3038         int desc_index,
3039         int *desc_length)
3040 {
3041         int ret;
3042         u8 header[QUERY_DESC_HDR_SIZE];
3043         int header_len = QUERY_DESC_HDR_SIZE;
3044
3045         if (desc_id >= QUERY_DESC_IDN_MAX)
3046                 return -EINVAL;
3047
3048         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3049                                         desc_id, desc_index, 0, header,
3050                                         &header_len);
3051
3052         if (ret) {
3053                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
3054                         __func__, desc_id);
3055                 return ret;
3056         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3057                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3058                         __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3059                         desc_id);
3060                 ret = -EINVAL;
3061         }
3062
3063         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3064         return ret;
3065
3066 }
3067
3068 /**
3069  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3070  * @hba: Pointer to adapter instance
3071  * @desc_id: descriptor idn value
3072  * @desc_len: mapped desc length (out)
3073  *
3074  * Return 0 in case of success, non-zero otherwise
3075  */
3076 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3077         enum desc_idn desc_id, int *desc_len)
3078 {
3079         switch (desc_id) {
3080         case QUERY_DESC_IDN_DEVICE:
3081                 *desc_len = hba->desc_size.dev_desc;
3082                 break;
3083         case QUERY_DESC_IDN_POWER:
3084                 *desc_len = hba->desc_size.pwr_desc;
3085                 break;
3086         case QUERY_DESC_IDN_GEOMETRY:
3087                 *desc_len = hba->desc_size.geom_desc;
3088                 break;
3089         case QUERY_DESC_IDN_CONFIGURATION:
3090                 *desc_len = hba->desc_size.conf_desc;
3091                 break;
3092         case QUERY_DESC_IDN_UNIT:
3093                 *desc_len = hba->desc_size.unit_desc;
3094                 break;
3095         case QUERY_DESC_IDN_INTERCONNECT:
3096                 *desc_len = hba->desc_size.interc_desc;
3097                 break;
3098         case QUERY_DESC_IDN_STRING:
3099                 *desc_len = QUERY_DESC_MAX_SIZE;
3100                 break;
3101         case QUERY_DESC_IDN_HEALTH:
3102                 *desc_len = hba->desc_size.hlth_desc;
3103                 break;
3104         case QUERY_DESC_IDN_RFU_0:
3105         case QUERY_DESC_IDN_RFU_1:
3106                 *desc_len = 0;
3107                 break;
3108         default:
3109                 *desc_len = 0;
3110                 return -EINVAL;
3111         }
3112         return 0;
3113 }
3114 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3115
3116 /**
3117  * ufshcd_read_desc_param - read the specified descriptor parameter
3118  * @hba: Pointer to adapter instance
3119  * @desc_id: descriptor idn value
3120  * @desc_index: descriptor index
3121  * @param_offset: offset of the parameter to read
3122  * @param_read_buf: pointer to buffer where parameter would be read
3123  * @param_size: sizeof(param_read_buf)
3124  *
3125  * Return 0 in case of success, non-zero otherwise
3126  */
3127 int ufshcd_read_desc_param(struct ufs_hba *hba,
3128                            enum desc_idn desc_id,
3129                            int desc_index,
3130                            u8 param_offset,
3131                            u8 *param_read_buf,
3132                            u8 param_size)
3133 {
3134         int ret;
3135         u8 *desc_buf;
3136         int buff_len;
3137         bool is_kmalloc = true;
3138
3139         /* Safety check */
3140         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3141                 return -EINVAL;
3142
3143         /* Get the max length of descriptor from structure filled up at probe
3144          * time.
3145          */
3146         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3147
3148         /* Sanity checks */
3149         if (ret || !buff_len) {
3150                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3151                         __func__);
3152                 return ret;
3153         }
3154
3155         /* Check whether we need temp memory */
3156         if (param_offset != 0 || param_size < buff_len) {
3157                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3158                 if (!desc_buf)
3159                         return -ENOMEM;
3160         } else {
3161                 desc_buf = param_read_buf;
3162                 is_kmalloc = false;
3163         }
3164
3165         /* Request for full descriptor */
3166         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3167                                         desc_id, desc_index, 0,
3168                                         desc_buf, &buff_len);
3169
3170         if (ret) {
3171                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3172                         __func__, desc_id, desc_index, param_offset, ret);
3173                 goto out;
3174         }
3175
3176         /* Sanity check */
3177         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3178                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3179                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3180                 ret = -EINVAL;
3181                 goto out;
3182         }
3183
3184         /* Check wherher we will not copy more data, than available */
3185         if (is_kmalloc && param_size > buff_len)
3186                 param_size = buff_len;
3187
3188         if (is_kmalloc)
3189                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3190 out:
3191         if (is_kmalloc)
3192                 kfree(desc_buf);
3193         return ret;
3194 }
3195
3196 static inline int ufshcd_read_desc(struct ufs_hba *hba,
3197                                    enum desc_idn desc_id,
3198                                    int desc_index,
3199                                    u8 *buf,
3200                                    u32 size)
3201 {
3202         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3203 }
3204
3205 static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3206                                          u8 *buf,
3207                                          u32 size)
3208 {
3209         return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
3210 }
3211
3212 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
3213 {
3214         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3215 }
3216
3217 /**
3218  * ufshcd_read_string_desc - read string descriptor
3219  * @hba: pointer to adapter instance
3220  * @desc_index: descriptor index
3221  * @buf: pointer to buffer where descriptor would be read
3222  * @size: size of buf
3223  * @ascii: if true convert from unicode to ascii characters
3224  *
3225  * Return 0 in case of success, non-zero otherwise
3226  */
3227 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
3228                             u8 *buf, u32 size, bool ascii)
3229 {
3230         int err = 0;
3231
3232         err = ufshcd_read_desc(hba,
3233                                 QUERY_DESC_IDN_STRING, desc_index, buf, size);
3234
3235         if (err) {
3236                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
3237                         __func__, QUERY_REQ_RETRIES, err);
3238                 goto out;
3239         }
3240
3241         if (ascii) {
3242                 int desc_len;
3243                 int ascii_len;
3244                 int i;
3245                 char *buff_ascii;
3246
3247                 desc_len = buf[0];
3248                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3249                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3250                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
3251                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
3252                                         __func__);
3253                         err = -ENOMEM;
3254                         goto out;
3255                 }
3256
3257                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
3258                 if (!buff_ascii) {
3259                         err = -ENOMEM;
3260                         goto out;
3261                 }
3262
3263                 /*
3264                  * the descriptor contains string in UTF16 format
3265                  * we need to convert to utf-8 so it can be displayed
3266                  */
3267                 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
3268                                 desc_len - QUERY_DESC_HDR_SIZE,
3269                                 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
3270
3271                 /* replace non-printable or non-ASCII characters with spaces */
3272                 for (i = 0; i < ascii_len; i++)
3273                         ufshcd_remove_non_printable(&buff_ascii[i]);
3274
3275                 memset(buf + QUERY_DESC_HDR_SIZE, 0,