Merge tag 'riscv-for-linus-4.20-rc4' of git://git.kernel.org/pub/scm/linux/kernel...
[sfrench/cifs-2.6.git] / drivers / scsi / ufs / ufshcd.c
1 /*
2  * Universal Flash Storage Host controller driver Core
3  *
4  * This code is based on drivers/scsi/ufs/ufshcd.c
5  * Copyright (C) 2011-2013 Samsung India Software Operations
6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7  *
8  * Authors:
9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
10  *      Vinayak Holikatti <h.vinayak@samsung.com>
11  *
12  * This program is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU General Public License
14  * as published by the Free Software Foundation; either version 2
15  * of the License, or (at your option) any later version.
16  * See the COPYING file in the top-level directory or visit
17  * <http://www.gnu.org/licenses/gpl-2.0.html>
18  *
19  * This program is distributed in the hope that it will be useful,
20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22  * GNU General Public License for more details.
23  *
24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
25  * without warranty of any kind. You are solely responsible for
26  * determining the appropriateness of using and distributing
27  * the program and assume all risks associated with your exercise
28  * of rights with respect to the program, including but not limited
29  * to infringement of third party rights, the risks and costs of
30  * program errors, damage to or loss of data, programs or equipment,
31  * and unavailability or interruption of operations. Under no
32  * circumstances will the contributor of this Program be liable for
33  * any damages of any kind arising from your use or distribution of
34  * this program.
35  *
36  * The Linux Foundation chooses to take subject only to the GPLv2
37  * license terms, and distributes only under these terms.
38  */
39
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
45 #include "ufshcd.h"
46 #include "ufs_quirks.h"
47 #include "unipro.h"
48 #include "ufs-sysfs.h"
49 #include "ufs_bsg.h"
50
51 #define CREATE_TRACE_POINTS
52 #include <trace/events/ufs.h>
53
54 #define UFSHCD_REQ_SENSE_SIZE   18
55
56 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
57                                  UTP_TASK_REQ_COMPL |\
58                                  UFSHCD_ERROR_MASK)
59 /* UIC command timeout, unit: ms */
60 #define UIC_CMD_TIMEOUT 500
61
62 /* NOP OUT retries waiting for NOP IN response */
63 #define NOP_OUT_RETRIES    10
64 /* Timeout after 30 msecs if NOP OUT hangs without response */
65 #define NOP_OUT_TIMEOUT    30 /* msecs */
66
67 /* Query request retries */
68 #define QUERY_REQ_RETRIES 3
69 /* Query request timeout */
70 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
71
72 /* Task management command timeout */
73 #define TM_CMD_TIMEOUT  100 /* msecs */
74
75 /* maximum number of retries for a general UIC command  */
76 #define UFS_UIC_COMMAND_RETRIES 3
77
78 /* maximum number of link-startup retries */
79 #define DME_LINKSTARTUP_RETRIES 3
80
81 /* Maximum retries for Hibern8 enter */
82 #define UIC_HIBERN8_ENTER_RETRIES 3
83
84 /* maximum number of reset retries before giving up */
85 #define MAX_HOST_RESET_RETRIES 5
86
87 /* Expose the flag value from utp_upiu_query.value */
88 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
89
90 /* Interrupt aggregation default timeout, unit: 40us */
91 #define INT_AGGR_DEF_TO 0x02
92
93 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
94         ({                                                              \
95                 int _ret;                                               \
96                 if (_on)                                                \
97                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
98                 else                                                    \
99                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
100                 _ret;                                                   \
101         })
102
103 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
104         size_t __len = (len);                                            \
105         print_hex_dump(KERN_ERR, prefix_str,                             \
106                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
107                        16, 4, buf, __len, false);                        \
108 } while (0)
109
110 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
111                      const char *prefix)
112 {
113         u8 *regs;
114
115         regs = kzalloc(len, GFP_KERNEL);
116         if (!regs)
117                 return -ENOMEM;
118
119         memcpy_fromio(regs, hba->mmio_base + offset, len);
120         ufshcd_hex_dump(prefix, regs, len);
121         kfree(regs);
122
123         return 0;
124 }
125 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
126
127 enum {
128         UFSHCD_MAX_CHANNEL      = 0,
129         UFSHCD_MAX_ID           = 1,
130         UFSHCD_CMD_PER_LUN      = 32,
131         UFSHCD_CAN_QUEUE        = 32,
132 };
133
134 /* UFSHCD states */
135 enum {
136         UFSHCD_STATE_RESET,
137         UFSHCD_STATE_ERROR,
138         UFSHCD_STATE_OPERATIONAL,
139         UFSHCD_STATE_EH_SCHEDULED,
140 };
141
142 /* UFSHCD error handling flags */
143 enum {
144         UFSHCD_EH_IN_PROGRESS = (1 << 0),
145 };
146
147 /* UFSHCD UIC layer error flags */
148 enum {
149         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
150         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
151         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
152         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
153         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
154         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
155 };
156
157 #define ufshcd_set_eh_in_progress(h) \
158         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
159 #define ufshcd_eh_in_progress(h) \
160         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
161 #define ufshcd_clear_eh_in_progress(h) \
162         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
163
164 #define ufshcd_set_ufs_dev_active(h) \
165         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
166 #define ufshcd_set_ufs_dev_sleep(h) \
167         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
168 #define ufshcd_set_ufs_dev_poweroff(h) \
169         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
170 #define ufshcd_is_ufs_dev_active(h) \
171         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
172 #define ufshcd_is_ufs_dev_sleep(h) \
173         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
174 #define ufshcd_is_ufs_dev_poweroff(h) \
175         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
176
177 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
178         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
179         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
180         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
181         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
182         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
183         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
184 };
185
186 static inline enum ufs_dev_pwr_mode
187 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
188 {
189         return ufs_pm_lvl_states[lvl].dev_state;
190 }
191
192 static inline enum uic_link_state
193 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
194 {
195         return ufs_pm_lvl_states[lvl].link_state;
196 }
197
198 static inline enum ufs_pm_level
199 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
200                                         enum uic_link_state link_state)
201 {
202         enum ufs_pm_level lvl;
203
204         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
205                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
206                         (ufs_pm_lvl_states[lvl].link_state == link_state))
207                         return lvl;
208         }
209
210         /* if no match found, return the level 0 */
211         return UFS_PM_LVL_0;
212 }
213
214 static struct ufs_dev_fix ufs_fixups[] = {
215         /* UFS cards deviations table */
216         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
217                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
218         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
219         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
220                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
221         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
222                 UFS_DEVICE_NO_FASTAUTO),
223         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
224                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
225         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
226                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
227         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
228                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
229         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
230                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
231         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
232         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
233                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
234
235         END_FIX
236 };
237
238 static void ufshcd_tmc_handler(struct ufs_hba *hba);
239 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
240 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
241 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
242 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
243 static void ufshcd_hba_exit(struct ufs_hba *hba);
244 static int ufshcd_probe_hba(struct ufs_hba *hba);
245 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
246                                  bool skip_ref_clk);
247 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
248 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
249 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
250 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
251 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
252 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
253 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
254 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
255 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
256 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
257 static irqreturn_t ufshcd_intr(int irq, void *__hba);
258 static int ufshcd_change_power_mode(struct ufs_hba *hba,
259                              struct ufs_pa_layer_attr *pwr_mode);
260 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
261 {
262         return tag >= 0 && tag < hba->nutrs;
263 }
264
265 static inline int ufshcd_enable_irq(struct ufs_hba *hba)
266 {
267         int ret = 0;
268
269         if (!hba->is_irq_enabled) {
270                 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
271                                 hba);
272                 if (ret)
273                         dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
274                                 __func__, ret);
275                 hba->is_irq_enabled = true;
276         }
277
278         return ret;
279 }
280
281 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
282 {
283         if (hba->is_irq_enabled) {
284                 free_irq(hba->irq, hba);
285                 hba->is_irq_enabled = false;
286         }
287 }
288
289 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
290 {
291         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
292                 scsi_unblock_requests(hba->host);
293 }
294
295 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
296 {
297         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
298                 scsi_block_requests(hba->host);
299 }
300
301 /* replace non-printable or non-ASCII characters with spaces */
302 static inline void ufshcd_remove_non_printable(char *val)
303 {
304         if (!val)
305                 return;
306
307         if (*val < 0x20 || *val > 0x7e)
308                 *val = ' ';
309 }
310
311 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
312                 const char *str)
313 {
314         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
315
316         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
317 }
318
319 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
320                 const char *str)
321 {
322         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
323
324         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
325 }
326
327 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
328                 const char *str)
329 {
330         int off = (int)tag - hba->nutrs;
331         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
332
333         trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
334                         &descp->input_param1);
335 }
336
337 static void ufshcd_add_command_trace(struct ufs_hba *hba,
338                 unsigned int tag, const char *str)
339 {
340         sector_t lba = -1;
341         u8 opcode = 0;
342         u32 intr, doorbell;
343         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
344         int transfer_len = -1;
345
346         if (!trace_ufshcd_command_enabled()) {
347                 /* trace UPIU W/O tracing command */
348                 if (lrbp->cmd)
349                         ufshcd_add_cmd_upiu_trace(hba, tag, str);
350                 return;
351         }
352
353         if (lrbp->cmd) { /* data phase exists */
354                 /* trace UPIU also */
355                 ufshcd_add_cmd_upiu_trace(hba, tag, str);
356                 opcode = (u8)(*lrbp->cmd->cmnd);
357                 if ((opcode == READ_10) || (opcode == WRITE_10)) {
358                         /*
359                          * Currently we only fully trace read(10) and write(10)
360                          * commands
361                          */
362                         if (lrbp->cmd->request && lrbp->cmd->request->bio)
363                                 lba =
364                                   lrbp->cmd->request->bio->bi_iter.bi_sector;
365                         transfer_len = be32_to_cpu(
366                                 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
367                 }
368         }
369
370         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
371         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
372         trace_ufshcd_command(dev_name(hba->dev), str, tag,
373                                 doorbell, transfer_len, intr, lba, opcode);
374 }
375
376 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
377 {
378         struct ufs_clk_info *clki;
379         struct list_head *head = &hba->clk_list_head;
380
381         if (list_empty(head))
382                 return;
383
384         list_for_each_entry(clki, head, list) {
385                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
386                                 clki->max_freq)
387                         dev_err(hba->dev, "clk: %s, rate: %u\n",
388                                         clki->name, clki->curr_freq);
389         }
390 }
391
392 static void ufshcd_print_uic_err_hist(struct ufs_hba *hba,
393                 struct ufs_uic_err_reg_hist *err_hist, char *err_name)
394 {
395         int i;
396
397         for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) {
398                 int p = (i + err_hist->pos - 1) % UIC_ERR_REG_HIST_LENGTH;
399
400                 if (err_hist->reg[p] == 0)
401                         continue;
402                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i,
403                         err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
404         }
405 }
406
407 static void ufshcd_print_host_regs(struct ufs_hba *hba)
408 {
409         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
410         dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
411                 hba->ufs_version, hba->capabilities);
412         dev_err(hba->dev,
413                 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
414                 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
415         dev_err(hba->dev,
416                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
417                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
418                 hba->ufs_stats.hibern8_exit_cnt);
419
420         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
421         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
422         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
423         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
424         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
425
426         ufshcd_print_clk_freqs(hba);
427
428         if (hba->vops && hba->vops->dbg_register_dump)
429                 hba->vops->dbg_register_dump(hba);
430 }
431
432 static
433 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
434 {
435         struct ufshcd_lrb *lrbp;
436         int prdt_length;
437         int tag;
438
439         for_each_set_bit(tag, &bitmap, hba->nutrs) {
440                 lrbp = &hba->lrb[tag];
441
442                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
443                                 tag, ktime_to_us(lrbp->issue_time_stamp));
444                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
445                                 tag, ktime_to_us(lrbp->compl_time_stamp));
446                 dev_err(hba->dev,
447                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
448                         tag, (u64)lrbp->utrd_dma_addr);
449
450                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
451                                 sizeof(struct utp_transfer_req_desc));
452                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
453                         (u64)lrbp->ucd_req_dma_addr);
454                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
455                                 sizeof(struct utp_upiu_req));
456                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
457                         (u64)lrbp->ucd_rsp_dma_addr);
458                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
459                                 sizeof(struct utp_upiu_rsp));
460
461                 prdt_length = le16_to_cpu(
462                         lrbp->utr_descriptor_ptr->prd_table_length);
463                 dev_err(hba->dev,
464                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
465                         tag, prdt_length,
466                         (u64)lrbp->ucd_prdt_dma_addr);
467
468                 if (pr_prdt)
469                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
470                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
471         }
472 }
473
474 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
475 {
476         int tag;
477
478         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
479                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
480
481                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
482                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
483         }
484 }
485
486 static void ufshcd_print_host_state(struct ufs_hba *hba)
487 {
488         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
489         dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
490                 hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
491         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
492                 hba->saved_err, hba->saved_uic_err);
493         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
494                 hba->curr_dev_pwr_mode, hba->uic_link_state);
495         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
496                 hba->pm_op_in_progress, hba->is_sys_suspended);
497         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
498                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
499         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
500         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
501                 hba->eh_flags, hba->req_abort_count);
502         dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
503                 hba->capabilities, hba->caps);
504         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
505                 hba->dev_quirks);
506 }
507
508 /**
509  * ufshcd_print_pwr_info - print power params as saved in hba
510  * power info
511  * @hba: per-adapter instance
512  */
513 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
514 {
515         static const char * const names[] = {
516                 "INVALID MODE",
517                 "FAST MODE",
518                 "SLOW_MODE",
519                 "INVALID MODE",
520                 "FASTAUTO_MODE",
521                 "SLOWAUTO_MODE",
522                 "INVALID MODE",
523         };
524
525         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
526                  __func__,
527                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
528                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
529                  names[hba->pwr_info.pwr_rx],
530                  names[hba->pwr_info.pwr_tx],
531                  hba->pwr_info.hs_rate);
532 }
533
534 /*
535  * ufshcd_wait_for_register - wait for register value to change
536  * @hba - per-adapter interface
537  * @reg - mmio register offset
538  * @mask - mask to apply to read register value
539  * @val - wait condition
540  * @interval_us - polling interval in microsecs
541  * @timeout_ms - timeout in millisecs
542  * @can_sleep - perform sleep or just spin
543  *
544  * Returns -ETIMEDOUT on error, zero on success
545  */
546 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
547                                 u32 val, unsigned long interval_us,
548                                 unsigned long timeout_ms, bool can_sleep)
549 {
550         int err = 0;
551         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
552
553         /* ignore bits that we don't intend to wait on */
554         val = val & mask;
555
556         while ((ufshcd_readl(hba, reg) & mask) != val) {
557                 if (can_sleep)
558                         usleep_range(interval_us, interval_us + 50);
559                 else
560                         udelay(interval_us);
561                 if (time_after(jiffies, timeout)) {
562                         if ((ufshcd_readl(hba, reg) & mask) != val)
563                                 err = -ETIMEDOUT;
564                         break;
565                 }
566         }
567
568         return err;
569 }
570
571 /**
572  * ufshcd_get_intr_mask - Get the interrupt bit mask
573  * @hba: Pointer to adapter instance
574  *
575  * Returns interrupt bit mask per version
576  */
577 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
578 {
579         u32 intr_mask = 0;
580
581         switch (hba->ufs_version) {
582         case UFSHCI_VERSION_10:
583                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
584                 break;
585         case UFSHCI_VERSION_11:
586         case UFSHCI_VERSION_20:
587                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
588                 break;
589         case UFSHCI_VERSION_21:
590         default:
591                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
592                 break;
593         }
594
595         return intr_mask;
596 }
597
598 /**
599  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
600  * @hba: Pointer to adapter instance
601  *
602  * Returns UFSHCI version supported by the controller
603  */
604 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
605 {
606         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
607                 return ufshcd_vops_get_ufs_hci_version(hba);
608
609         return ufshcd_readl(hba, REG_UFS_VERSION);
610 }
611
612 /**
613  * ufshcd_is_device_present - Check if any device connected to
614  *                            the host controller
615  * @hba: pointer to adapter instance
616  *
617  * Returns true if device present, false if no device detected
618  */
619 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
620 {
621         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
622                                                 DEVICE_PRESENT) ? true : false;
623 }
624
625 /**
626  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
627  * @lrbp: pointer to local command reference block
628  *
629  * This function is used to get the OCS field from UTRD
630  * Returns the OCS field in the UTRD
631  */
632 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
633 {
634         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
635 }
636
637 /**
638  * ufshcd_get_tm_free_slot - get a free slot for task management request
639  * @hba: per adapter instance
640  * @free_slot: pointer to variable with available slot value
641  *
642  * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
643  * Returns 0 if free slot is not available, else return 1 with tag value
644  * in @free_slot.
645  */
646 static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
647 {
648         int tag;
649         bool ret = false;
650
651         if (!free_slot)
652                 goto out;
653
654         do {
655                 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
656                 if (tag >= hba->nutmrs)
657                         goto out;
658         } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
659
660         *free_slot = tag;
661         ret = true;
662 out:
663         return ret;
664 }
665
666 static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
667 {
668         clear_bit_unlock(slot, &hba->tm_slots_in_use);
669 }
670
671 /**
672  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
673  * @hba: per adapter instance
674  * @pos: position of the bit to be cleared
675  */
676 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
677 {
678         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
679                 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
680         else
681                 ufshcd_writel(hba, ~(1 << pos),
682                                 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
683 }
684
685 /**
686  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
687  * @hba: per adapter instance
688  * @pos: position of the bit to be cleared
689  */
690 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
691 {
692         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
693                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
694         else
695                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
696 }
697
698 /**
699  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
700  * @hba: per adapter instance
701  * @tag: position of the bit to be cleared
702  */
703 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
704 {
705         __clear_bit(tag, &hba->outstanding_reqs);
706 }
707
708 /**
709  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
710  * @reg: Register value of host controller status
711  *
712  * Returns integer, 0 on Success and positive value if failed
713  */
714 static inline int ufshcd_get_lists_status(u32 reg)
715 {
716         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
717 }
718
719 /**
720  * ufshcd_get_uic_cmd_result - Get the UIC command result
721  * @hba: Pointer to adapter instance
722  *
723  * This function gets the result of UIC command completion
724  * Returns 0 on success, non zero value on error
725  */
726 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
727 {
728         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
729                MASK_UIC_COMMAND_RESULT;
730 }
731
732 /**
733  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
734  * @hba: Pointer to adapter instance
735  *
736  * This function gets UIC command argument3
737  * Returns 0 on success, non zero value on error
738  */
739 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
740 {
741         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
742 }
743
744 /**
745  * ufshcd_get_req_rsp - returns the TR response transaction type
746  * @ucd_rsp_ptr: pointer to response UPIU
747  */
748 static inline int
749 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
750 {
751         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
752 }
753
754 /**
755  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
756  * @ucd_rsp_ptr: pointer to response UPIU
757  *
758  * This function gets the response status and scsi_status from response UPIU
759  * Returns the response result code.
760  */
761 static inline int
762 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
763 {
764         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
765 }
766
767 /*
768  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
769  *                              from response UPIU
770  * @ucd_rsp_ptr: pointer to response UPIU
771  *
772  * Return the data segment length.
773  */
774 static inline unsigned int
775 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
776 {
777         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
778                 MASK_RSP_UPIU_DATA_SEG_LEN;
779 }
780
781 /**
782  * ufshcd_is_exception_event - Check if the device raised an exception event
783  * @ucd_rsp_ptr: pointer to response UPIU
784  *
785  * The function checks if the device raised an exception event indicated in
786  * the Device Information field of response UPIU.
787  *
788  * Returns true if exception is raised, false otherwise.
789  */
790 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
791 {
792         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
793                         MASK_RSP_EXCEPTION_EVENT ? true : false;
794 }
795
796 /**
797  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
798  * @hba: per adapter instance
799  */
800 static inline void
801 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
802 {
803         ufshcd_writel(hba, INT_AGGR_ENABLE |
804                       INT_AGGR_COUNTER_AND_TIMER_RESET,
805                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
806 }
807
808 /**
809  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
810  * @hba: per adapter instance
811  * @cnt: Interrupt aggregation counter threshold
812  * @tmout: Interrupt aggregation timeout value
813  */
814 static inline void
815 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
816 {
817         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
818                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
819                       INT_AGGR_TIMEOUT_VAL(tmout),
820                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
821 }
822
823 /**
824  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
825  * @hba: per adapter instance
826  */
827 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
828 {
829         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
830 }
831
832 /**
833  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
834  *                      When run-stop registers are set to 1, it indicates the
835  *                      host controller that it can process the requests
836  * @hba: per adapter instance
837  */
838 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
839 {
840         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
841                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
842         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
843                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
844 }
845
846 /**
847  * ufshcd_hba_start - Start controller initialization sequence
848  * @hba: per adapter instance
849  */
850 static inline void ufshcd_hba_start(struct ufs_hba *hba)
851 {
852         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
853 }
854
855 /**
856  * ufshcd_is_hba_active - Get controller state
857  * @hba: per adapter instance
858  *
859  * Returns false if controller is active, true otherwise
860  */
861 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
862 {
863         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
864                 ? false : true;
865 }
866
867 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
868 {
869         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
870         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
871             (hba->ufs_version == UFSHCI_VERSION_11))
872                 return UFS_UNIPRO_VER_1_41;
873         else
874                 return UFS_UNIPRO_VER_1_6;
875 }
876 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
877
878 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
879 {
880         /*
881          * If both host and device support UniPro ver1.6 or later, PA layer
882          * parameters tuning happens during link startup itself.
883          *
884          * We can manually tune PA layer parameters if either host or device
885          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
886          * logic simple, we will only do manual tuning if local unipro version
887          * doesn't support ver1.6 or later.
888          */
889         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
890                 return true;
891         else
892                 return false;
893 }
894
895 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
896 {
897         int ret = 0;
898         struct ufs_clk_info *clki;
899         struct list_head *head = &hba->clk_list_head;
900         ktime_t start = ktime_get();
901         bool clk_state_changed = false;
902
903         if (list_empty(head))
904                 goto out;
905
906         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
907         if (ret)
908                 return ret;
909
910         list_for_each_entry(clki, head, list) {
911                 if (!IS_ERR_OR_NULL(clki->clk)) {
912                         if (scale_up && clki->max_freq) {
913                                 if (clki->curr_freq == clki->max_freq)
914                                         continue;
915
916                                 clk_state_changed = true;
917                                 ret = clk_set_rate(clki->clk, clki->max_freq);
918                                 if (ret) {
919                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
920                                                 __func__, clki->name,
921                                                 clki->max_freq, ret);
922                                         break;
923                                 }
924                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
925                                                 "scaled up", clki->name,
926                                                 clki->curr_freq,
927                                                 clki->max_freq);
928
929                                 clki->curr_freq = clki->max_freq;
930
931                         } else if (!scale_up && clki->min_freq) {
932                                 if (clki->curr_freq == clki->min_freq)
933                                         continue;
934
935                                 clk_state_changed = true;
936                                 ret = clk_set_rate(clki->clk, clki->min_freq);
937                                 if (ret) {
938                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
939                                                 __func__, clki->name,
940                                                 clki->min_freq, ret);
941                                         break;
942                                 }
943                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
944                                                 "scaled down", clki->name,
945                                                 clki->curr_freq,
946                                                 clki->min_freq);
947                                 clki->curr_freq = clki->min_freq;
948                         }
949                 }
950                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
951                                 clki->name, clk_get_rate(clki->clk));
952         }
953
954         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
955
956 out:
957         if (clk_state_changed)
958                 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
959                         (scale_up ? "up" : "down"),
960                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
961         return ret;
962 }
963
964 /**
965  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
966  * @hba: per adapter instance
967  * @scale_up: True if scaling up and false if scaling down
968  *
969  * Returns true if scaling is required, false otherwise.
970  */
971 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
972                                                bool scale_up)
973 {
974         struct ufs_clk_info *clki;
975         struct list_head *head = &hba->clk_list_head;
976
977         if (list_empty(head))
978                 return false;
979
980         list_for_each_entry(clki, head, list) {
981                 if (!IS_ERR_OR_NULL(clki->clk)) {
982                         if (scale_up && clki->max_freq) {
983                                 if (clki->curr_freq == clki->max_freq)
984                                         continue;
985                                 return true;
986                         } else if (!scale_up && clki->min_freq) {
987                                 if (clki->curr_freq == clki->min_freq)
988                                         continue;
989                                 return true;
990                         }
991                 }
992         }
993
994         return false;
995 }
996
997 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
998                                         u64 wait_timeout_us)
999 {
1000         unsigned long flags;
1001         int ret = 0;
1002         u32 tm_doorbell;
1003         u32 tr_doorbell;
1004         bool timeout = false, do_last_check = false;
1005         ktime_t start;
1006
1007         ufshcd_hold(hba, false);
1008         spin_lock_irqsave(hba->host->host_lock, flags);
1009         /*
1010          * Wait for all the outstanding tasks/transfer requests.
1011          * Verify by checking the doorbell registers are clear.
1012          */
1013         start = ktime_get();
1014         do {
1015                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1016                         ret = -EBUSY;
1017                         goto out;
1018                 }
1019
1020                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1021                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1022                 if (!tm_doorbell && !tr_doorbell) {
1023                         timeout = false;
1024                         break;
1025                 } else if (do_last_check) {
1026                         break;
1027                 }
1028
1029                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1030                 schedule();
1031                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1032                     wait_timeout_us) {
1033                         timeout = true;
1034                         /*
1035                          * We might have scheduled out for long time so make
1036                          * sure to check if doorbells are cleared by this time
1037                          * or not.
1038                          */
1039                         do_last_check = true;
1040                 }
1041                 spin_lock_irqsave(hba->host->host_lock, flags);
1042         } while (tm_doorbell || tr_doorbell);
1043
1044         if (timeout) {
1045                 dev_err(hba->dev,
1046                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1047                         __func__, tm_doorbell, tr_doorbell);
1048                 ret = -EBUSY;
1049         }
1050 out:
1051         spin_unlock_irqrestore(hba->host->host_lock, flags);
1052         ufshcd_release(hba);
1053         return ret;
1054 }
1055
1056 /**
1057  * ufshcd_scale_gear - scale up/down UFS gear
1058  * @hba: per adapter instance
1059  * @scale_up: True for scaling up gear and false for scaling down
1060  *
1061  * Returns 0 for success,
1062  * Returns -EBUSY if scaling can't happen at this time
1063  * Returns non-zero for any other errors
1064  */
1065 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1066 {
1067         #define UFS_MIN_GEAR_TO_SCALE_DOWN      UFS_HS_G1
1068         int ret = 0;
1069         struct ufs_pa_layer_attr new_pwr_info;
1070
1071         if (scale_up) {
1072                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1073                        sizeof(struct ufs_pa_layer_attr));
1074         } else {
1075                 memcpy(&new_pwr_info, &hba->pwr_info,
1076                        sizeof(struct ufs_pa_layer_attr));
1077
1078                 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1079                     || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1080                         /* save the current power mode */
1081                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1082                                 &hba->pwr_info,
1083                                 sizeof(struct ufs_pa_layer_attr));
1084
1085                         /* scale down gear */
1086                         new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1087                         new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1088                 }
1089         }
1090
1091         /* check if the power mode needs to be changed or not? */
1092         ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1093
1094         if (ret)
1095                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1096                         __func__, ret,
1097                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1098                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1099
1100         return ret;
1101 }
1102
1103 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1104 {
1105         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1106         int ret = 0;
1107         /*
1108          * make sure that there are no outstanding requests when
1109          * clock scaling is in progress
1110          */
1111         ufshcd_scsi_block_requests(hba);
1112         down_write(&hba->clk_scaling_lock);
1113         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1114                 ret = -EBUSY;
1115                 up_write(&hba->clk_scaling_lock);
1116                 ufshcd_scsi_unblock_requests(hba);
1117         }
1118
1119         return ret;
1120 }
1121
1122 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1123 {
1124         up_write(&hba->clk_scaling_lock);
1125         ufshcd_scsi_unblock_requests(hba);
1126 }
1127
1128 /**
1129  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1130  * @hba: per adapter instance
1131  * @scale_up: True for scaling up and false for scalin down
1132  *
1133  * Returns 0 for success,
1134  * Returns -EBUSY if scaling can't happen at this time
1135  * Returns non-zero for any other errors
1136  */
1137 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1138 {
1139         int ret = 0;
1140
1141         /* let's not get into low power until clock scaling is completed */
1142         ufshcd_hold(hba, false);
1143
1144         ret = ufshcd_clock_scaling_prepare(hba);
1145         if (ret)
1146                 return ret;
1147
1148         /* scale down the gear before scaling down clocks */
1149         if (!scale_up) {
1150                 ret = ufshcd_scale_gear(hba, false);
1151                 if (ret)
1152                         goto out;
1153         }
1154
1155         ret = ufshcd_scale_clks(hba, scale_up);
1156         if (ret) {
1157                 if (!scale_up)
1158                         ufshcd_scale_gear(hba, true);
1159                 goto out;
1160         }
1161
1162         /* scale up the gear after scaling up clocks */
1163         if (scale_up) {
1164                 ret = ufshcd_scale_gear(hba, true);
1165                 if (ret) {
1166                         ufshcd_scale_clks(hba, false);
1167                         goto out;
1168                 }
1169         }
1170
1171         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1172
1173 out:
1174         ufshcd_clock_scaling_unprepare(hba);
1175         ufshcd_release(hba);
1176         return ret;
1177 }
1178
1179 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1180 {
1181         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1182                                            clk_scaling.suspend_work);
1183         unsigned long irq_flags;
1184
1185         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1186         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1187                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1188                 return;
1189         }
1190         hba->clk_scaling.is_suspended = true;
1191         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1192
1193         __ufshcd_suspend_clkscaling(hba);
1194 }
1195
1196 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1197 {
1198         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1199                                            clk_scaling.resume_work);
1200         unsigned long irq_flags;
1201
1202         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1203         if (!hba->clk_scaling.is_suspended) {
1204                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1205                 return;
1206         }
1207         hba->clk_scaling.is_suspended = false;
1208         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1209
1210         devfreq_resume_device(hba->devfreq);
1211 }
1212
1213 static int ufshcd_devfreq_target(struct device *dev,
1214                                 unsigned long *freq, u32 flags)
1215 {
1216         int ret = 0;
1217         struct ufs_hba *hba = dev_get_drvdata(dev);
1218         ktime_t start;
1219         bool scale_up, sched_clk_scaling_suspend_work = false;
1220         struct list_head *clk_list = &hba->clk_list_head;
1221         struct ufs_clk_info *clki;
1222         unsigned long irq_flags;
1223
1224         if (!ufshcd_is_clkscaling_supported(hba))
1225                 return -EINVAL;
1226
1227         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1228         if (ufshcd_eh_in_progress(hba)) {
1229                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1230                 return 0;
1231         }
1232
1233         if (!hba->clk_scaling.active_reqs)
1234                 sched_clk_scaling_suspend_work = true;
1235
1236         if (list_empty(clk_list)) {
1237                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1238                 goto out;
1239         }
1240
1241         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1242         scale_up = (*freq == clki->max_freq) ? true : false;
1243         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1244                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1245                 ret = 0;
1246                 goto out; /* no state change required */
1247         }
1248         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1249
1250         start = ktime_get();
1251         ret = ufshcd_devfreq_scale(hba, scale_up);
1252
1253         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1254                 (scale_up ? "up" : "down"),
1255                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1256
1257 out:
1258         if (sched_clk_scaling_suspend_work)
1259                 queue_work(hba->clk_scaling.workq,
1260                            &hba->clk_scaling.suspend_work);
1261
1262         return ret;
1263 }
1264
1265
1266 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1267                 struct devfreq_dev_status *stat)
1268 {
1269         struct ufs_hba *hba = dev_get_drvdata(dev);
1270         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1271         unsigned long flags;
1272
1273         if (!ufshcd_is_clkscaling_supported(hba))
1274                 return -EINVAL;
1275
1276         memset(stat, 0, sizeof(*stat));
1277
1278         spin_lock_irqsave(hba->host->host_lock, flags);
1279         if (!scaling->window_start_t)
1280                 goto start_window;
1281
1282         if (scaling->is_busy_started)
1283                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1284                                         scaling->busy_start_t));
1285
1286         stat->total_time = jiffies_to_usecs((long)jiffies -
1287                                 (long)scaling->window_start_t);
1288         stat->busy_time = scaling->tot_busy_t;
1289 start_window:
1290         scaling->window_start_t = jiffies;
1291         scaling->tot_busy_t = 0;
1292
1293         if (hba->outstanding_reqs) {
1294                 scaling->busy_start_t = ktime_get();
1295                 scaling->is_busy_started = true;
1296         } else {
1297                 scaling->busy_start_t = 0;
1298                 scaling->is_busy_started = false;
1299         }
1300         spin_unlock_irqrestore(hba->host->host_lock, flags);
1301         return 0;
1302 }
1303
1304 static struct devfreq_dev_profile ufs_devfreq_profile = {
1305         .polling_ms     = 100,
1306         .target         = ufshcd_devfreq_target,
1307         .get_dev_status = ufshcd_devfreq_get_dev_status,
1308 };
1309
1310 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1311 {
1312         struct list_head *clk_list = &hba->clk_list_head;
1313         struct ufs_clk_info *clki;
1314         struct devfreq *devfreq;
1315         int ret;
1316
1317         /* Skip devfreq if we don't have any clocks in the list */
1318         if (list_empty(clk_list))
1319                 return 0;
1320
1321         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1322         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1323         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1324
1325         devfreq = devfreq_add_device(hba->dev,
1326                         &ufs_devfreq_profile,
1327                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1328                         NULL);
1329         if (IS_ERR(devfreq)) {
1330                 ret = PTR_ERR(devfreq);
1331                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1332
1333                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1334                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1335                 return ret;
1336         }
1337
1338         hba->devfreq = devfreq;
1339
1340         return 0;
1341 }
1342
1343 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1344 {
1345         struct list_head *clk_list = &hba->clk_list_head;
1346         struct ufs_clk_info *clki;
1347
1348         if (!hba->devfreq)
1349                 return;
1350
1351         devfreq_remove_device(hba->devfreq);
1352         hba->devfreq = NULL;
1353
1354         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1355         dev_pm_opp_remove(hba->dev, clki->min_freq);
1356         dev_pm_opp_remove(hba->dev, clki->max_freq);
1357 }
1358
1359 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1360 {
1361         unsigned long flags;
1362
1363         devfreq_suspend_device(hba->devfreq);
1364         spin_lock_irqsave(hba->host->host_lock, flags);
1365         hba->clk_scaling.window_start_t = 0;
1366         spin_unlock_irqrestore(hba->host->host_lock, flags);
1367 }
1368
1369 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1370 {
1371         unsigned long flags;
1372         bool suspend = false;
1373
1374         if (!ufshcd_is_clkscaling_supported(hba))
1375                 return;
1376
1377         spin_lock_irqsave(hba->host->host_lock, flags);
1378         if (!hba->clk_scaling.is_suspended) {
1379                 suspend = true;
1380                 hba->clk_scaling.is_suspended = true;
1381         }
1382         spin_unlock_irqrestore(hba->host->host_lock, flags);
1383
1384         if (suspend)
1385                 __ufshcd_suspend_clkscaling(hba);
1386 }
1387
1388 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1389 {
1390         unsigned long flags;
1391         bool resume = false;
1392
1393         if (!ufshcd_is_clkscaling_supported(hba))
1394                 return;
1395
1396         spin_lock_irqsave(hba->host->host_lock, flags);
1397         if (hba->clk_scaling.is_suspended) {
1398                 resume = true;
1399                 hba->clk_scaling.is_suspended = false;
1400         }
1401         spin_unlock_irqrestore(hba->host->host_lock, flags);
1402
1403         if (resume)
1404                 devfreq_resume_device(hba->devfreq);
1405 }
1406
1407 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1408                 struct device_attribute *attr, char *buf)
1409 {
1410         struct ufs_hba *hba = dev_get_drvdata(dev);
1411
1412         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1413 }
1414
1415 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1416                 struct device_attribute *attr, const char *buf, size_t count)
1417 {
1418         struct ufs_hba *hba = dev_get_drvdata(dev);
1419         u32 value;
1420         int err;
1421
1422         if (kstrtou32(buf, 0, &value))
1423                 return -EINVAL;
1424
1425         value = !!value;
1426         if (value == hba->clk_scaling.is_allowed)
1427                 goto out;
1428
1429         pm_runtime_get_sync(hba->dev);
1430         ufshcd_hold(hba, false);
1431
1432         cancel_work_sync(&hba->clk_scaling.suspend_work);
1433         cancel_work_sync(&hba->clk_scaling.resume_work);
1434
1435         hba->clk_scaling.is_allowed = value;
1436
1437         if (value) {
1438                 ufshcd_resume_clkscaling(hba);
1439         } else {
1440                 ufshcd_suspend_clkscaling(hba);
1441                 err = ufshcd_devfreq_scale(hba, true);
1442                 if (err)
1443                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1444                                         __func__, err);
1445         }
1446
1447         ufshcd_release(hba);
1448         pm_runtime_put_sync(hba->dev);
1449 out:
1450         return count;
1451 }
1452
1453 static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1454 {
1455         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1456         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1457         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1458         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1459         hba->clk_scaling.enable_attr.attr.mode = 0644;
1460         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1461                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1462 }
1463
1464 static void ufshcd_ungate_work(struct work_struct *work)
1465 {
1466         int ret;
1467         unsigned long flags;
1468         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1469                         clk_gating.ungate_work);
1470
1471         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1472
1473         spin_lock_irqsave(hba->host->host_lock, flags);
1474         if (hba->clk_gating.state == CLKS_ON) {
1475                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1476                 goto unblock_reqs;
1477         }
1478
1479         spin_unlock_irqrestore(hba->host->host_lock, flags);
1480         ufshcd_setup_clocks(hba, true);
1481
1482         /* Exit from hibern8 */
1483         if (ufshcd_can_hibern8_during_gating(hba)) {
1484                 /* Prevent gating in this path */
1485                 hba->clk_gating.is_suspended = true;
1486                 if (ufshcd_is_link_hibern8(hba)) {
1487                         ret = ufshcd_uic_hibern8_exit(hba);
1488                         if (ret)
1489                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1490                                         __func__, ret);
1491                         else
1492                                 ufshcd_set_link_active(hba);
1493                 }
1494                 hba->clk_gating.is_suspended = false;
1495         }
1496 unblock_reqs:
1497         ufshcd_scsi_unblock_requests(hba);
1498 }
1499
1500 /**
1501  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1502  * Also, exit from hibern8 mode and set the link as active.
1503  * @hba: per adapter instance
1504  * @async: This indicates whether caller should ungate clocks asynchronously.
1505  */
1506 int ufshcd_hold(struct ufs_hba *hba, bool async)
1507 {
1508         int rc = 0;
1509         unsigned long flags;
1510
1511         if (!ufshcd_is_clkgating_allowed(hba))
1512                 goto out;
1513         spin_lock_irqsave(hba->host->host_lock, flags);
1514         hba->clk_gating.active_reqs++;
1515
1516         if (ufshcd_eh_in_progress(hba)) {
1517                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1518                 return 0;
1519         }
1520
1521 start:
1522         switch (hba->clk_gating.state) {
1523         case CLKS_ON:
1524                 /*
1525                  * Wait for the ungate work to complete if in progress.
1526                  * Though the clocks may be in ON state, the link could
1527                  * still be in hibner8 state if hibern8 is allowed
1528                  * during clock gating.
1529                  * Make sure we exit hibern8 state also in addition to
1530                  * clocks being ON.
1531                  */
1532                 if (ufshcd_can_hibern8_during_gating(hba) &&
1533                     ufshcd_is_link_hibern8(hba)) {
1534                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1535                         flush_work(&hba->clk_gating.ungate_work);
1536                         spin_lock_irqsave(hba->host->host_lock, flags);
1537                         goto start;
1538                 }
1539                 break;
1540         case REQ_CLKS_OFF:
1541                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1542                         hba->clk_gating.state = CLKS_ON;
1543                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1544                                                 hba->clk_gating.state);
1545                         break;
1546                 }
1547                 /*
1548                  * If we are here, it means gating work is either done or
1549                  * currently running. Hence, fall through to cancel gating
1550                  * work and to enable clocks.
1551                  */
1552         case CLKS_OFF:
1553                 ufshcd_scsi_block_requests(hba);
1554                 hba->clk_gating.state = REQ_CLKS_ON;
1555                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1556                                         hba->clk_gating.state);
1557                 queue_work(hba->clk_gating.clk_gating_workq,
1558                            &hba->clk_gating.ungate_work);
1559                 /*
1560                  * fall through to check if we should wait for this
1561                  * work to be done or not.
1562                  */
1563         case REQ_CLKS_ON:
1564                 if (async) {
1565                         rc = -EAGAIN;
1566                         hba->clk_gating.active_reqs--;
1567                         break;
1568                 }
1569
1570                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1571                 flush_work(&hba->clk_gating.ungate_work);
1572                 /* Make sure state is CLKS_ON before returning */
1573                 spin_lock_irqsave(hba->host->host_lock, flags);
1574                 goto start;
1575         default:
1576                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1577                                 __func__, hba->clk_gating.state);
1578                 break;
1579         }
1580         spin_unlock_irqrestore(hba->host->host_lock, flags);
1581 out:
1582         return rc;
1583 }
1584 EXPORT_SYMBOL_GPL(ufshcd_hold);
1585
1586 static void ufshcd_gate_work(struct work_struct *work)
1587 {
1588         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1589                         clk_gating.gate_work.work);
1590         unsigned long flags;
1591
1592         spin_lock_irqsave(hba->host->host_lock, flags);
1593         /*
1594          * In case you are here to cancel this work the gating state
1595          * would be marked as REQ_CLKS_ON. In this case save time by
1596          * skipping the gating work and exit after changing the clock
1597          * state to CLKS_ON.
1598          */
1599         if (hba->clk_gating.is_suspended ||
1600                 (hba->clk_gating.state == REQ_CLKS_ON)) {
1601                 hba->clk_gating.state = CLKS_ON;
1602                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1603                                         hba->clk_gating.state);
1604                 goto rel_lock;
1605         }
1606
1607         if (hba->clk_gating.active_reqs
1608                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1609                 || hba->lrb_in_use || hba->outstanding_tasks
1610                 || hba->active_uic_cmd || hba->uic_async_done)
1611                 goto rel_lock;
1612
1613         spin_unlock_irqrestore(hba->host->host_lock, flags);
1614
1615         /* put the link into hibern8 mode before turning off clocks */
1616         if (ufshcd_can_hibern8_during_gating(hba)) {
1617                 if (ufshcd_uic_hibern8_enter(hba)) {
1618                         hba->clk_gating.state = CLKS_ON;
1619                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1620                                                 hba->clk_gating.state);
1621                         goto out;
1622                 }
1623                 ufshcd_set_link_hibern8(hba);
1624         }
1625
1626         if (!ufshcd_is_link_active(hba))
1627                 ufshcd_setup_clocks(hba, false);
1628         else
1629                 /* If link is active, device ref_clk can't be switched off */
1630                 __ufshcd_setup_clocks(hba, false, true);
1631
1632         /*
1633          * In case you are here to cancel this work the gating state
1634          * would be marked as REQ_CLKS_ON. In this case keep the state
1635          * as REQ_CLKS_ON which would anyway imply that clocks are off
1636          * and a request to turn them on is pending. By doing this way,
1637          * we keep the state machine in tact and this would ultimately
1638          * prevent from doing cancel work multiple times when there are
1639          * new requests arriving before the current cancel work is done.
1640          */
1641         spin_lock_irqsave(hba->host->host_lock, flags);
1642         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1643                 hba->clk_gating.state = CLKS_OFF;
1644                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1645                                         hba->clk_gating.state);
1646         }
1647 rel_lock:
1648         spin_unlock_irqrestore(hba->host->host_lock, flags);
1649 out:
1650         return;
1651 }
1652
1653 /* host lock must be held before calling this variant */
1654 static void __ufshcd_release(struct ufs_hba *hba)
1655 {
1656         if (!ufshcd_is_clkgating_allowed(hba))
1657                 return;
1658
1659         hba->clk_gating.active_reqs--;
1660
1661         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1662                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1663                 || hba->lrb_in_use || hba->outstanding_tasks
1664                 || hba->active_uic_cmd || hba->uic_async_done
1665                 || ufshcd_eh_in_progress(hba))
1666                 return;
1667
1668         hba->clk_gating.state = REQ_CLKS_OFF;
1669         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1670         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1671                            &hba->clk_gating.gate_work,
1672                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1673 }
1674
1675 void ufshcd_release(struct ufs_hba *hba)
1676 {
1677         unsigned long flags;
1678
1679         spin_lock_irqsave(hba->host->host_lock, flags);
1680         __ufshcd_release(hba);
1681         spin_unlock_irqrestore(hba->host->host_lock, flags);
1682 }
1683 EXPORT_SYMBOL_GPL(ufshcd_release);
1684
1685 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1686                 struct device_attribute *attr, char *buf)
1687 {
1688         struct ufs_hba *hba = dev_get_drvdata(dev);
1689
1690         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1691 }
1692
1693 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1694                 struct device_attribute *attr, const char *buf, size_t count)
1695 {
1696         struct ufs_hba *hba = dev_get_drvdata(dev);
1697         unsigned long flags, value;
1698
1699         if (kstrtoul(buf, 0, &value))
1700                 return -EINVAL;
1701
1702         spin_lock_irqsave(hba->host->host_lock, flags);
1703         hba->clk_gating.delay_ms = value;
1704         spin_unlock_irqrestore(hba->host->host_lock, flags);
1705         return count;
1706 }
1707
1708 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1709                 struct device_attribute *attr, char *buf)
1710 {
1711         struct ufs_hba *hba = dev_get_drvdata(dev);
1712
1713         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1714 }
1715
1716 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1717                 struct device_attribute *attr, const char *buf, size_t count)
1718 {
1719         struct ufs_hba *hba = dev_get_drvdata(dev);
1720         unsigned long flags;
1721         u32 value;
1722
1723         if (kstrtou32(buf, 0, &value))
1724                 return -EINVAL;
1725
1726         value = !!value;
1727         if (value == hba->clk_gating.is_enabled)
1728                 goto out;
1729
1730         if (value) {
1731                 ufshcd_release(hba);
1732         } else {
1733                 spin_lock_irqsave(hba->host->host_lock, flags);
1734                 hba->clk_gating.active_reqs++;
1735                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1736         }
1737
1738         hba->clk_gating.is_enabled = value;
1739 out:
1740         return count;
1741 }
1742
1743 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1744 {
1745         char wq_name[sizeof("ufs_clkscaling_00")];
1746
1747         if (!ufshcd_is_clkscaling_supported(hba))
1748                 return;
1749
1750         INIT_WORK(&hba->clk_scaling.suspend_work,
1751                   ufshcd_clk_scaling_suspend_work);
1752         INIT_WORK(&hba->clk_scaling.resume_work,
1753                   ufshcd_clk_scaling_resume_work);
1754
1755         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1756                  hba->host->host_no);
1757         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1758
1759         ufshcd_clkscaling_init_sysfs(hba);
1760 }
1761
1762 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1763 {
1764         if (!ufshcd_is_clkscaling_supported(hba))
1765                 return;
1766
1767         destroy_workqueue(hba->clk_scaling.workq);
1768         ufshcd_devfreq_remove(hba);
1769 }
1770
1771 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1772 {
1773         char wq_name[sizeof("ufs_clk_gating_00")];
1774
1775         if (!ufshcd_is_clkgating_allowed(hba))
1776                 return;
1777
1778         hba->clk_gating.delay_ms = 150;
1779         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1780         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1781
1782         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1783                  hba->host->host_no);
1784         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1785                                                            WQ_MEM_RECLAIM);
1786
1787         hba->clk_gating.is_enabled = true;
1788
1789         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1790         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1791         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1792         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1793         hba->clk_gating.delay_attr.attr.mode = 0644;
1794         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1795                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1796
1797         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1798         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1799         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1800         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1801         hba->clk_gating.enable_attr.attr.mode = 0644;
1802         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1803                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1804 }
1805
1806 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1807 {
1808         if (!ufshcd_is_clkgating_allowed(hba))
1809                 return;
1810         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1811         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1812         cancel_work_sync(&hba->clk_gating.ungate_work);
1813         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1814         destroy_workqueue(hba->clk_gating.clk_gating_workq);
1815 }
1816
1817 /* Must be called with host lock acquired */
1818 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1819 {
1820         bool queue_resume_work = false;
1821
1822         if (!ufshcd_is_clkscaling_supported(hba))
1823                 return;
1824
1825         if (!hba->clk_scaling.active_reqs++)
1826                 queue_resume_work = true;
1827
1828         if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1829                 return;
1830
1831         if (queue_resume_work)
1832                 queue_work(hba->clk_scaling.workq,
1833                            &hba->clk_scaling.resume_work);
1834
1835         if (!hba->clk_scaling.window_start_t) {
1836                 hba->clk_scaling.window_start_t = jiffies;
1837                 hba->clk_scaling.tot_busy_t = 0;
1838                 hba->clk_scaling.is_busy_started = false;
1839         }
1840
1841         if (!hba->clk_scaling.is_busy_started) {
1842                 hba->clk_scaling.busy_start_t = ktime_get();
1843                 hba->clk_scaling.is_busy_started = true;
1844         }
1845 }
1846
1847 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1848 {
1849         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1850
1851         if (!ufshcd_is_clkscaling_supported(hba))
1852                 return;
1853
1854         if (!hba->outstanding_reqs && scaling->is_busy_started) {
1855                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1856                                         scaling->busy_start_t));
1857                 scaling->busy_start_t = 0;
1858                 scaling->is_busy_started = false;
1859         }
1860 }
1861 /**
1862  * ufshcd_send_command - Send SCSI or device management commands
1863  * @hba: per adapter instance
1864  * @task_tag: Task tag of the command
1865  */
1866 static inline
1867 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1868 {
1869         hba->lrb[task_tag].issue_time_stamp = ktime_get();
1870         hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
1871         ufshcd_clk_scaling_start_busy(hba);
1872         __set_bit(task_tag, &hba->outstanding_reqs);
1873         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1874         /* Make sure that doorbell is committed immediately */
1875         wmb();
1876         ufshcd_add_command_trace(hba, task_tag, "send");
1877 }
1878
1879 /**
1880  * ufshcd_copy_sense_data - Copy sense data in case of check condition
1881  * @lrbp: pointer to local reference block
1882  */
1883 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1884 {
1885         int len;
1886         if (lrbp->sense_buffer &&
1887             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1888                 int len_to_copy;
1889
1890                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1891                 len_to_copy = min_t(int, RESPONSE_UPIU_SENSE_DATA_LENGTH, len);
1892
1893                 memcpy(lrbp->sense_buffer,
1894                         lrbp->ucd_rsp_ptr->sr.sense_data,
1895                         min_t(int, len_to_copy, UFSHCD_REQ_SENSE_SIZE));
1896         }
1897 }
1898
1899 /**
1900  * ufshcd_copy_query_response() - Copy the Query Response and the data
1901  * descriptor
1902  * @hba: per adapter instance
1903  * @lrbp: pointer to local reference block
1904  */
1905 static
1906 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1907 {
1908         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1909
1910         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
1911
1912         /* Get the descriptor */
1913         if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
1914                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
1915                                 GENERAL_UPIU_REQUEST_SIZE;
1916                 u16 resp_len;
1917                 u16 buf_len;
1918
1919                 /* data segment length */
1920                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
1921                                                 MASK_QUERY_DATA_SEG_LEN;
1922                 buf_len = be16_to_cpu(
1923                                 hba->dev_cmd.query.request.upiu_req.length);
1924                 if (likely(buf_len >= resp_len)) {
1925                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1926                 } else {
1927                         dev_warn(hba->dev,
1928                                 "%s: Response size is bigger than buffer",
1929                                 __func__);
1930                         return -EINVAL;
1931                 }
1932         }
1933
1934         return 0;
1935 }
1936
1937 /**
1938  * ufshcd_hba_capabilities - Read controller capabilities
1939  * @hba: per adapter instance
1940  */
1941 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1942 {
1943         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1944
1945         /* nutrs and nutmrs are 0 based values */
1946         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
1947         hba->nutmrs =
1948         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
1949 }
1950
1951 /**
1952  * ufshcd_ready_for_uic_cmd - Check if controller is ready
1953  *                            to accept UIC commands
1954  * @hba: per adapter instance
1955  * Return true on success, else false
1956  */
1957 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
1958 {
1959         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
1960                 return true;
1961         else
1962                 return false;
1963 }
1964
1965 /**
1966  * ufshcd_get_upmcrs - Get the power mode change request status
1967  * @hba: Pointer to adapter instance
1968  *
1969  * This function gets the UPMCRS field of HCS register
1970  * Returns value of UPMCRS field
1971  */
1972 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
1973 {
1974         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
1975 }
1976
1977 /**
1978  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1979  * @hba: per adapter instance
1980  * @uic_cmd: UIC command
1981  *
1982  * Mutex must be held.
1983  */
1984 static inline void
1985 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1986 {
1987         WARN_ON(hba->active_uic_cmd);
1988
1989         hba->active_uic_cmd = uic_cmd;
1990
1991         /* Write Args */
1992         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
1993         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
1994         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
1995
1996         /* Write UIC Cmd */
1997         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
1998                       REG_UIC_COMMAND);
1999 }
2000
2001 /**
2002  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2003  * @hba: per adapter instance
2004  * @uic_cmd: UIC command
2005  *
2006  * Must be called with mutex held.
2007  * Returns 0 only if success.
2008  */
2009 static int
2010 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2011 {
2012         int ret;
2013         unsigned long flags;
2014
2015         if (wait_for_completion_timeout(&uic_cmd->done,
2016                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2017                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2018         else
2019                 ret = -ETIMEDOUT;
2020
2021         spin_lock_irqsave(hba->host->host_lock, flags);
2022         hba->active_uic_cmd = NULL;
2023         spin_unlock_irqrestore(hba->host->host_lock, flags);
2024
2025         return ret;
2026 }
2027
2028 /**
2029  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2030  * @hba: per adapter instance
2031  * @uic_cmd: UIC command
2032  * @completion: initialize the completion only if this is set to true
2033  *
2034  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2035  * with mutex held and host_lock locked.
2036  * Returns 0 only if success.
2037  */
2038 static int
2039 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2040                       bool completion)
2041 {
2042         if (!ufshcd_ready_for_uic_cmd(hba)) {
2043                 dev_err(hba->dev,
2044                         "Controller not ready to accept UIC commands\n");
2045                 return -EIO;
2046         }
2047
2048         if (completion)
2049                 init_completion(&uic_cmd->done);
2050
2051         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2052
2053         return 0;
2054 }
2055
2056 /**
2057  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2058  * @hba: per adapter instance
2059  * @uic_cmd: UIC command
2060  *
2061  * Returns 0 only if success.
2062  */
2063 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2064 {
2065         int ret;
2066         unsigned long flags;
2067
2068         ufshcd_hold(hba, false);
2069         mutex_lock(&hba->uic_cmd_mutex);
2070         ufshcd_add_delay_before_dme_cmd(hba);
2071
2072         spin_lock_irqsave(hba->host->host_lock, flags);
2073         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2074         spin_unlock_irqrestore(hba->host->host_lock, flags);
2075         if (!ret)
2076                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2077
2078         mutex_unlock(&hba->uic_cmd_mutex);
2079
2080         ufshcd_release(hba);
2081         return ret;
2082 }
2083
2084 /**
2085  * ufshcd_map_sg - Map scatter-gather list to prdt
2086  * @hba: per adapter instance
2087  * @lrbp: pointer to local reference block
2088  *
2089  * Returns 0 in case of success, non-zero value in case of failure
2090  */
2091 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2092 {
2093         struct ufshcd_sg_entry *prd_table;
2094         struct scatterlist *sg;
2095         struct scsi_cmnd *cmd;
2096         int sg_segments;
2097         int i;
2098
2099         cmd = lrbp->cmd;
2100         sg_segments = scsi_dma_map(cmd);
2101         if (sg_segments < 0)
2102                 return sg_segments;
2103
2104         if (sg_segments) {
2105                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2106                         lrbp->utr_descriptor_ptr->prd_table_length =
2107                                 cpu_to_le16((u16)(sg_segments *
2108                                         sizeof(struct ufshcd_sg_entry)));
2109                 else
2110                         lrbp->utr_descriptor_ptr->prd_table_length =
2111                                 cpu_to_le16((u16) (sg_segments));
2112
2113                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2114
2115                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2116                         prd_table[i].size  =
2117                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2118                         prd_table[i].base_addr =
2119                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2120                         prd_table[i].upper_addr =
2121                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2122                         prd_table[i].reserved = 0;
2123                 }
2124         } else {
2125                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2126         }
2127
2128         return 0;
2129 }
2130
2131 /**
2132  * ufshcd_enable_intr - enable interrupts
2133  * @hba: per adapter instance
2134  * @intrs: interrupt bits
2135  */
2136 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2137 {
2138         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2139
2140         if (hba->ufs_version == UFSHCI_VERSION_10) {
2141                 u32 rw;
2142                 rw = set & INTERRUPT_MASK_RW_VER_10;
2143                 set = rw | ((set ^ intrs) & intrs);
2144         } else {
2145                 set |= intrs;
2146         }
2147
2148         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2149 }
2150
2151 /**
2152  * ufshcd_disable_intr - disable interrupts
2153  * @hba: per adapter instance
2154  * @intrs: interrupt bits
2155  */
2156 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2157 {
2158         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2159
2160         if (hba->ufs_version == UFSHCI_VERSION_10) {
2161                 u32 rw;
2162                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2163                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2164                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2165
2166         } else {
2167                 set &= ~intrs;
2168         }
2169
2170         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2171 }
2172
2173 /**
2174  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2175  * descriptor according to request
2176  * @lrbp: pointer to local reference block
2177  * @upiu_flags: flags required in the header
2178  * @cmd_dir: requests data direction
2179  */
2180 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2181                         u32 *upiu_flags, enum dma_data_direction cmd_dir)
2182 {
2183         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2184         u32 data_direction;
2185         u32 dword_0;
2186
2187         if (cmd_dir == DMA_FROM_DEVICE) {
2188                 data_direction = UTP_DEVICE_TO_HOST;
2189                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2190         } else if (cmd_dir == DMA_TO_DEVICE) {
2191                 data_direction = UTP_HOST_TO_DEVICE;
2192                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2193         } else {
2194                 data_direction = UTP_NO_DATA_TRANSFER;
2195                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2196         }
2197
2198         dword_0 = data_direction | (lrbp->command_type
2199                                 << UPIU_COMMAND_TYPE_OFFSET);
2200         if (lrbp->intr_cmd)
2201                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2202
2203         /* Transfer request descriptor header fields */
2204         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2205         /* dword_1 is reserved, hence it is set to 0 */
2206         req_desc->header.dword_1 = 0;
2207         /*
2208          * assigning invalid value for command status. Controller
2209          * updates OCS on command completion, with the command
2210          * status
2211          */
2212         req_desc->header.dword_2 =
2213                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2214         /* dword_3 is reserved, hence it is set to 0 */
2215         req_desc->header.dword_3 = 0;
2216
2217         req_desc->prd_table_length = 0;
2218 }
2219
2220 /**
2221  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2222  * for scsi commands
2223  * @lrbp: local reference block pointer
2224  * @upiu_flags: flags
2225  */
2226 static
2227 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2228 {
2229         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2230         unsigned short cdb_len;
2231
2232         /* command descriptor fields */
2233         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2234                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2235                                 lrbp->lun, lrbp->task_tag);
2236         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2237                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2238
2239         /* Total EHS length and Data segment length will be zero */
2240         ucd_req_ptr->header.dword_2 = 0;
2241
2242         ucd_req_ptr->sc.exp_data_transfer_len =
2243                 cpu_to_be32(lrbp->cmd->sdb.length);
2244
2245         cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, UFS_CDB_SIZE);
2246         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2247         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2248
2249         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2250 }
2251
2252 /**
2253  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2254  * for query requsts
2255  * @hba: UFS hba
2256  * @lrbp: local reference block pointer
2257  * @upiu_flags: flags
2258  */
2259 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2260                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2261 {
2262         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2263         struct ufs_query *query = &hba->dev_cmd.query;
2264         u16 len = be16_to_cpu(query->request.upiu_req.length);
2265
2266         /* Query request header */
2267         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2268                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2269                         lrbp->lun, lrbp->task_tag);
2270         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2271                         0, query->request.query_func, 0, 0);
2272
2273         /* Data segment length only need for WRITE_DESC */
2274         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2275                 ucd_req_ptr->header.dword_2 =
2276                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2277         else
2278                 ucd_req_ptr->header.dword_2 = 0;
2279
2280         /* Copy the Query Request buffer as is */
2281         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2282                         QUERY_OSF_SIZE);
2283
2284         /* Copy the Descriptor */
2285         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2286                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2287
2288         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2289 }
2290
2291 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2292 {
2293         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2294
2295         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2296
2297         /* command descriptor fields */
2298         ucd_req_ptr->header.dword_0 =
2299                 UPIU_HEADER_DWORD(
2300                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2301         /* clear rest of the fields of basic header */
2302         ucd_req_ptr->header.dword_1 = 0;
2303         ucd_req_ptr->header.dword_2 = 0;
2304
2305         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2306 }
2307
2308 /**
2309  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2310  *                           for Device Management Purposes
2311  * @hba: per adapter instance
2312  * @lrbp: pointer to local reference block
2313  */
2314 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2315 {
2316         u32 upiu_flags;
2317         int ret = 0;
2318
2319         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2320             (hba->ufs_version == UFSHCI_VERSION_11))
2321                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2322         else
2323                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2324
2325         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2326         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2327                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2328         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2329                 ufshcd_prepare_utp_nop_upiu(lrbp);
2330         else
2331                 ret = -EINVAL;
2332
2333         return ret;
2334 }
2335
2336 /**
2337  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2338  *                         for SCSI Purposes
2339  * @hba: per adapter instance
2340  * @lrbp: pointer to local reference block
2341  */
2342 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2343 {
2344         u32 upiu_flags;
2345         int ret = 0;
2346
2347         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2348             (hba->ufs_version == UFSHCI_VERSION_11))
2349                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2350         else
2351                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2352
2353         if (likely(lrbp->cmd)) {
2354                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2355                                                 lrbp->cmd->sc_data_direction);
2356                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2357         } else {
2358                 ret = -EINVAL;
2359         }
2360
2361         return ret;
2362 }
2363
2364 /**
2365  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2366  * @upiu_wlun_id: UPIU W-LUN id
2367  *
2368  * Returns SCSI W-LUN id
2369  */
2370 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2371 {
2372         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2373 }
2374
2375 /**
2376  * ufshcd_queuecommand - main entry point for SCSI requests
2377  * @host: SCSI host pointer
2378  * @cmd: command from SCSI Midlayer
2379  *
2380  * Returns 0 for success, non-zero in case of failure
2381  */
2382 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2383 {
2384         struct ufshcd_lrb *lrbp;
2385         struct ufs_hba *hba;
2386         unsigned long flags;
2387         int tag;
2388         int err = 0;
2389
2390         hba = shost_priv(host);
2391
2392         tag = cmd->request->tag;
2393         if (!ufshcd_valid_tag(hba, tag)) {
2394                 dev_err(hba->dev,
2395                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2396                         __func__, tag, cmd, cmd->request);
2397                 BUG();
2398         }
2399
2400         if (!down_read_trylock(&hba->clk_scaling_lock))
2401                 return SCSI_MLQUEUE_HOST_BUSY;
2402
2403         spin_lock_irqsave(hba->host->host_lock, flags);
2404         switch (hba->ufshcd_state) {
2405         case UFSHCD_STATE_OPERATIONAL:
2406                 break;
2407         case UFSHCD_STATE_EH_SCHEDULED:
2408         case UFSHCD_STATE_RESET:
2409                 err = SCSI_MLQUEUE_HOST_BUSY;
2410                 goto out_unlock;
2411         case UFSHCD_STATE_ERROR:
2412                 set_host_byte(cmd, DID_ERROR);
2413                 cmd->scsi_done(cmd);
2414                 goto out_unlock;
2415         default:
2416                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2417                                 __func__, hba->ufshcd_state);
2418                 set_host_byte(cmd, DID_BAD_TARGET);
2419                 cmd->scsi_done(cmd);
2420                 goto out_unlock;
2421         }
2422
2423         /* if error handling is in progress, don't issue commands */
2424         if (ufshcd_eh_in_progress(hba)) {
2425                 set_host_byte(cmd, DID_ERROR);
2426                 cmd->scsi_done(cmd);
2427                 goto out_unlock;
2428         }
2429         spin_unlock_irqrestore(hba->host->host_lock, flags);
2430
2431         hba->req_abort_count = 0;
2432
2433         /* acquire the tag to make sure device cmds don't use it */
2434         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
2435                 /*
2436                  * Dev manage command in progress, requeue the command.
2437                  * Requeuing the command helps in cases where the request *may*
2438                  * find different tag instead of waiting for dev manage command
2439                  * completion.
2440                  */
2441                 err = SCSI_MLQUEUE_HOST_BUSY;
2442                 goto out;
2443         }
2444
2445         err = ufshcd_hold(hba, true);
2446         if (err) {
2447                 err = SCSI_MLQUEUE_HOST_BUSY;
2448                 clear_bit_unlock(tag, &hba->lrb_in_use);
2449                 goto out;
2450         }
2451         WARN_ON(hba->clk_gating.state != CLKS_ON);
2452
2453         lrbp = &hba->lrb[tag];
2454
2455         WARN_ON(lrbp->cmd);
2456         lrbp->cmd = cmd;
2457         lrbp->sense_bufflen = UFSHCD_REQ_SENSE_SIZE;
2458         lrbp->sense_buffer = cmd->sense_buffer;
2459         lrbp->task_tag = tag;
2460         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2461         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2462         lrbp->req_abort_skip = false;
2463
2464         ufshcd_comp_scsi_upiu(hba, lrbp);
2465
2466         err = ufshcd_map_sg(hba, lrbp);
2467         if (err) {
2468                 lrbp->cmd = NULL;
2469                 clear_bit_unlock(tag, &hba->lrb_in_use);
2470                 goto out;
2471         }
2472         /* Make sure descriptors are ready before ringing the doorbell */
2473         wmb();
2474
2475         /* issue command to the controller */
2476         spin_lock_irqsave(hba->host->host_lock, flags);
2477         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2478         ufshcd_send_command(hba, tag);
2479 out_unlock:
2480         spin_unlock_irqrestore(hba->host->host_lock, flags);
2481 out:
2482         up_read(&hba->clk_scaling_lock);
2483         return err;
2484 }
2485
2486 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2487                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2488 {
2489         lrbp->cmd = NULL;
2490         lrbp->sense_bufflen = 0;
2491         lrbp->sense_buffer = NULL;
2492         lrbp->task_tag = tag;
2493         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2494         lrbp->intr_cmd = true; /* No interrupt aggregation */
2495         hba->dev_cmd.type = cmd_type;
2496
2497         return ufshcd_comp_devman_upiu(hba, lrbp);
2498 }
2499
2500 static int
2501 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2502 {
2503         int err = 0;
2504         unsigned long flags;
2505         u32 mask = 1 << tag;
2506
2507         /* clear outstanding transaction before retry */
2508         spin_lock_irqsave(hba->host->host_lock, flags);
2509         ufshcd_utrl_clear(hba, tag);
2510         spin_unlock_irqrestore(hba->host->host_lock, flags);
2511
2512         /*
2513          * wait for for h/w to clear corresponding bit in door-bell.
2514          * max. wait is 1 sec.
2515          */
2516         err = ufshcd_wait_for_register(hba,
2517                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2518                         mask, ~mask, 1000, 1000, true);
2519
2520         return err;
2521 }
2522
2523 static int
2524 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2525 {
2526         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2527
2528         /* Get the UPIU response */
2529         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2530                                 UPIU_RSP_CODE_OFFSET;
2531         return query_res->response;
2532 }
2533
2534 /**
2535  * ufshcd_dev_cmd_completion() - handles device management command responses
2536  * @hba: per adapter instance
2537  * @lrbp: pointer to local reference block
2538  */
2539 static int
2540 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2541 {
2542         int resp;
2543         int err = 0;
2544
2545         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2546         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2547
2548         switch (resp) {
2549         case UPIU_TRANSACTION_NOP_IN:
2550                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2551                         err = -EINVAL;
2552                         dev_err(hba->dev, "%s: unexpected response %x\n",
2553                                         __func__, resp);
2554                 }
2555                 break;
2556         case UPIU_TRANSACTION_QUERY_RSP:
2557                 err = ufshcd_check_query_response(hba, lrbp);
2558                 if (!err)
2559                         err = ufshcd_copy_query_response(hba, lrbp);
2560                 break;
2561         case UPIU_TRANSACTION_REJECT_UPIU:
2562                 /* TODO: handle Reject UPIU Response */
2563                 err = -EPERM;
2564                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2565                                 __func__);
2566                 break;
2567         default:
2568                 err = -EINVAL;
2569                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2570                                 __func__, resp);
2571                 break;
2572         }
2573
2574         return err;
2575 }
2576
2577 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2578                 struct ufshcd_lrb *lrbp, int max_timeout)
2579 {
2580         int err = 0;
2581         unsigned long time_left;
2582         unsigned long flags;
2583
2584         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2585                         msecs_to_jiffies(max_timeout));
2586
2587         /* Make sure descriptors are ready before ringing the doorbell */
2588         wmb();
2589         spin_lock_irqsave(hba->host->host_lock, flags);
2590         hba->dev_cmd.complete = NULL;
2591         if (likely(time_left)) {
2592                 err = ufshcd_get_tr_ocs(lrbp);
2593                 if (!err)
2594                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2595         }
2596         spin_unlock_irqrestore(hba->host->host_lock, flags);
2597
2598         if (!time_left) {
2599                 err = -ETIMEDOUT;
2600                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2601                         __func__, lrbp->task_tag);
2602                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2603                         /* successfully cleared the command, retry if needed */
2604                         err = -EAGAIN;
2605                 /*
2606                  * in case of an error, after clearing the doorbell,
2607                  * we also need to clear the outstanding_request
2608                  * field in hba
2609                  */
2610                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2611         }
2612
2613         return err;
2614 }
2615
2616 /**
2617  * ufshcd_get_dev_cmd_tag - Get device management command tag
2618  * @hba: per-adapter instance
2619  * @tag_out: pointer to variable with available slot value
2620  *
2621  * Get a free slot and lock it until device management command
2622  * completes.
2623  *
2624  * Returns false if free slot is unavailable for locking, else
2625  * return true with tag value in @tag.
2626  */
2627 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
2628 {
2629         int tag;
2630         bool ret = false;
2631         unsigned long tmp;
2632
2633         if (!tag_out)
2634                 goto out;
2635
2636         do {
2637                 tmp = ~hba->lrb_in_use;
2638                 tag = find_last_bit(&tmp, hba->nutrs);
2639                 if (tag >= hba->nutrs)
2640                         goto out;
2641         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
2642
2643         *tag_out = tag;
2644         ret = true;
2645 out:
2646         return ret;
2647 }
2648
2649 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
2650 {
2651         clear_bit_unlock(tag, &hba->lrb_in_use);
2652 }
2653
2654 /**
2655  * ufshcd_exec_dev_cmd - API for sending device management requests
2656  * @hba: UFS hba
2657  * @cmd_type: specifies the type (NOP, Query...)
2658  * @timeout: time in seconds
2659  *
2660  * NOTE: Since there is only one available tag for device management commands,
2661  * it is expected you hold the hba->dev_cmd.lock mutex.
2662  */
2663 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2664                 enum dev_cmd_type cmd_type, int timeout)
2665 {
2666         struct ufshcd_lrb *lrbp;
2667         int err;
2668         int tag;
2669         struct completion wait;
2670         unsigned long flags;
2671
2672         down_read(&hba->clk_scaling_lock);
2673
2674         /*
2675          * Get free slot, sleep if slots are unavailable.
2676          * Even though we use wait_event() which sleeps indefinitely,
2677          * the maximum wait time is bounded by SCSI request timeout.
2678          */
2679         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
2680
2681         init_completion(&wait);
2682         lrbp = &hba->lrb[tag];
2683         WARN_ON(lrbp->cmd);
2684         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2685         if (unlikely(err))
2686                 goto out_put_tag;
2687
2688         hba->dev_cmd.complete = &wait;
2689
2690         ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2691         /* Make sure descriptors are ready before ringing the doorbell */
2692         wmb();
2693         spin_lock_irqsave(hba->host->host_lock, flags);
2694         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2695         ufshcd_send_command(hba, tag);
2696         spin_unlock_irqrestore(hba->host->host_lock, flags);
2697
2698         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2699
2700         ufshcd_add_query_upiu_trace(hba, tag,
2701                         err ? "query_complete_err" : "query_complete");
2702
2703 out_put_tag:
2704         ufshcd_put_dev_cmd_tag(hba, tag);
2705         wake_up(&hba->dev_cmd.tag_wq);
2706         up_read(&hba->clk_scaling_lock);
2707         return err;
2708 }
2709
2710 /**
2711  * ufshcd_init_query() - init the query response and request parameters
2712  * @hba: per-adapter instance
2713  * @request: address of the request pointer to be initialized
2714  * @response: address of the response pointer to be initialized
2715  * @opcode: operation to perform
2716  * @idn: flag idn to access
2717  * @index: LU number to access
2718  * @selector: query/flag/descriptor further identification
2719  */
2720 static inline void ufshcd_init_query(struct ufs_hba *hba,
2721                 struct ufs_query_req **request, struct ufs_query_res **response,
2722                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2723 {
2724         *request = &hba->dev_cmd.query.request;
2725         *response = &hba->dev_cmd.query.response;
2726         memset(*request, 0, sizeof(struct ufs_query_req));
2727         memset(*response, 0, sizeof(struct ufs_query_res));
2728         (*request)->upiu_req.opcode = opcode;
2729         (*request)->upiu_req.idn = idn;
2730         (*request)->upiu_req.index = index;
2731         (*request)->upiu_req.selector = selector;
2732 }
2733
2734 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2735         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2736 {
2737         int ret;
2738         int retries;
2739
2740         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2741                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2742                 if (ret)
2743                         dev_dbg(hba->dev,
2744                                 "%s: failed with error %d, retries %d\n",
2745                                 __func__, ret, retries);
2746                 else
2747                         break;
2748         }
2749
2750         if (ret)
2751                 dev_err(hba->dev,
2752                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2753                         __func__, opcode, idn, ret, retries);
2754         return ret;
2755 }
2756
2757 /**
2758  * ufshcd_query_flag() - API function for sending flag query requests
2759  * @hba: per-adapter instance
2760  * @opcode: flag query to perform
2761  * @idn: flag idn to access
2762  * @flag_res: the flag value after the query request completes
2763  *
2764  * Returns 0 for success, non-zero in case of failure
2765  */
2766 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2767                         enum flag_idn idn, bool *flag_res)
2768 {
2769         struct ufs_query_req *request = NULL;
2770         struct ufs_query_res *response = NULL;
2771         int err, index = 0, selector = 0;
2772         int timeout = QUERY_REQ_TIMEOUT;
2773
2774         BUG_ON(!hba);
2775
2776         ufshcd_hold(hba, false);
2777         mutex_lock(&hba->dev_cmd.lock);
2778         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2779                         selector);
2780
2781         switch (opcode) {
2782         case UPIU_QUERY_OPCODE_SET_FLAG:
2783         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2784         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2785                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2786                 break;
2787         case UPIU_QUERY_OPCODE_READ_FLAG:
2788                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2789                 if (!flag_res) {
2790                         /* No dummy reads */
2791                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
2792                                         __func__);
2793                         err = -EINVAL;
2794                         goto out_unlock;
2795                 }
2796                 break;
2797         default:
2798                 dev_err(hba->dev,
2799                         "%s: Expected query flag opcode but got = %d\n",
2800                         __func__, opcode);
2801                 err = -EINVAL;
2802                 goto out_unlock;
2803         }
2804
2805         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2806
2807         if (err) {
2808                 dev_err(hba->dev,
2809                         "%s: Sending flag query for idn %d failed, err = %d\n",
2810                         __func__, idn, err);
2811                 goto out_unlock;
2812         }
2813
2814         if (flag_res)
2815                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2816                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2817
2818 out_unlock:
2819         mutex_unlock(&hba->dev_cmd.lock);
2820         ufshcd_release(hba);
2821         return err;
2822 }
2823
2824 /**
2825  * ufshcd_query_attr - API function for sending attribute requests
2826  * @hba: per-adapter instance
2827  * @opcode: attribute opcode
2828  * @idn: attribute idn to access
2829  * @index: index field
2830  * @selector: selector field
2831  * @attr_val: the attribute value after the query request completes
2832  *
2833  * Returns 0 for success, non-zero in case of failure
2834 */
2835 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2836                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2837 {
2838         struct ufs_query_req *request = NULL;
2839         struct ufs_query_res *response = NULL;
2840         int err;
2841
2842         BUG_ON(!hba);
2843
2844         ufshcd_hold(hba, false);
2845         if (!attr_val) {
2846                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2847                                 __func__, opcode);
2848                 err = -EINVAL;
2849                 goto out;
2850         }
2851
2852         mutex_lock(&hba->dev_cmd.lock);
2853         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2854                         selector);
2855
2856         switch (opcode) {
2857         case UPIU_QUERY_OPCODE_WRITE_ATTR:
2858                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2859                 request->upiu_req.value = cpu_to_be32(*attr_val);
2860                 break;
2861         case UPIU_QUERY_OPCODE_READ_ATTR:
2862                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2863                 break;
2864         default:
2865                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2866                                 __func__, opcode);
2867                 err = -EINVAL;
2868                 goto out_unlock;
2869         }
2870
2871         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2872
2873         if (err) {
2874                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2875                                 __func__, opcode, idn, index, err);
2876                 goto out_unlock;
2877         }
2878
2879         *attr_val = be32_to_cpu(response->upiu_res.value);
2880
2881 out_unlock:
2882         mutex_unlock(&hba->dev_cmd.lock);
2883 out:
2884         ufshcd_release(hba);
2885         return err;
2886 }
2887
2888 /**
2889  * ufshcd_query_attr_retry() - API function for sending query
2890  * attribute with retries
2891  * @hba: per-adapter instance
2892  * @opcode: attribute opcode
2893  * @idn: attribute idn to access
2894  * @index: index field
2895  * @selector: selector field
2896  * @attr_val: the attribute value after the query request
2897  * completes
2898  *
2899  * Returns 0 for success, non-zero in case of failure
2900 */
2901 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2902         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2903         u32 *attr_val)
2904 {
2905         int ret = 0;
2906         u32 retries;
2907
2908          for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2909                 ret = ufshcd_query_attr(hba, opcode, idn, index,
2910                                                 selector, attr_val);
2911                 if (ret)
2912                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2913                                 __func__, ret, retries);
2914                 else
2915                         break;
2916         }
2917
2918         if (ret)
2919                 dev_err(hba->dev,
2920                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2921                         __func__, idn, ret, QUERY_REQ_RETRIES);
2922         return ret;
2923 }
2924
2925 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
2926                         enum query_opcode opcode, enum desc_idn idn, u8 index,
2927                         u8 selector, u8 *desc_buf, int *buf_len)
2928 {
2929         struct ufs_query_req *request = NULL;
2930         struct ufs_query_res *response = NULL;
2931         int err;
2932
2933         BUG_ON(!hba);
2934
2935         ufshcd_hold(hba, false);
2936         if (!desc_buf) {
2937                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2938                                 __func__, opcode);
2939                 err = -EINVAL;
2940                 goto out;
2941         }
2942
2943         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
2944                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
2945                                 __func__, *buf_len);
2946                 err = -EINVAL;
2947                 goto out;
2948         }
2949
2950         mutex_lock(&hba->dev_cmd.lock);
2951         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2952                         selector);
2953         hba->dev_cmd.query.descriptor = desc_buf;
2954         request->upiu_req.length = cpu_to_be16(*buf_len);
2955
2956         switch (opcode) {
2957         case UPIU_QUERY_OPCODE_WRITE_DESC:
2958                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2959                 break;
2960         case UPIU_QUERY_OPCODE_READ_DESC:
2961                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2962                 break;
2963         default:
2964                 dev_err(hba->dev,
2965                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
2966                                 __func__, opcode);
2967                 err = -EINVAL;
2968                 goto out_unlock;
2969         }
2970
2971         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2972
2973         if (err) {
2974                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2975                                 __func__, opcode, idn, index, err);
2976                 goto out_unlock;
2977         }
2978
2979         hba->dev_cmd.query.descriptor = NULL;
2980         *buf_len = be16_to_cpu(response->upiu_res.length);
2981
2982 out_unlock:
2983         mutex_unlock(&hba->dev_cmd.lock);
2984 out:
2985         ufshcd_release(hba);
2986         return err;
2987 }
2988
2989 /**
2990  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2991  * @hba: per-adapter instance
2992  * @opcode: attribute opcode
2993  * @idn: attribute idn to access
2994  * @index: index field
2995  * @selector: selector field
2996  * @desc_buf: the buffer that contains the descriptor
2997  * @buf_len: length parameter passed to the device
2998  *
2999  * Returns 0 for success, non-zero in case of failure.
3000  * The buf_len parameter will contain, on return, the length parameter
3001  * received on the response.
3002  */
3003 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3004                                   enum query_opcode opcode,
3005                                   enum desc_idn idn, u8 index,
3006                                   u8 selector,
3007                                   u8 *desc_buf, int *buf_len)
3008 {
3009         int err;
3010         int retries;
3011
3012         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3013                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3014                                                 selector, desc_buf, buf_len);
3015                 if (!err || err == -EINVAL)
3016                         break;
3017         }
3018
3019         return err;
3020 }
3021
3022 /**
3023  * ufshcd_read_desc_length - read the specified descriptor length from header
3024  * @hba: Pointer to adapter instance
3025  * @desc_id: descriptor idn value
3026  * @desc_index: descriptor index
3027  * @desc_length: pointer to variable to read the length of descriptor
3028  *
3029  * Return 0 in case of success, non-zero otherwise
3030  */
3031 static int ufshcd_read_desc_length(struct ufs_hba *hba,
3032         enum desc_idn desc_id,
3033         int desc_index,
3034         int *desc_length)
3035 {
3036         int ret;
3037         u8 header[QUERY_DESC_HDR_SIZE];
3038         int header_len = QUERY_DESC_HDR_SIZE;
3039
3040         if (desc_id >= QUERY_DESC_IDN_MAX)
3041                 return -EINVAL;
3042
3043         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3044                                         desc_id, desc_index, 0, header,
3045                                         &header_len);
3046
3047         if (ret) {
3048                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
3049                         __func__, desc_id);
3050                 return ret;
3051         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3052                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3053                         __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3054                         desc_id);
3055                 ret = -EINVAL;
3056         }
3057
3058         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3059         return ret;
3060
3061 }
3062
3063 /**
3064  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3065  * @hba: Pointer to adapter instance
3066  * @desc_id: descriptor idn value
3067  * @desc_len: mapped desc length (out)
3068  *
3069  * Return 0 in case of success, non-zero otherwise
3070  */
3071 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3072         enum desc_idn desc_id, int *desc_len)
3073 {
3074         switch (desc_id) {
3075         case QUERY_DESC_IDN_DEVICE:
3076                 *desc_len = hba->desc_size.dev_desc;
3077                 break;
3078         case QUERY_DESC_IDN_POWER:
3079                 *desc_len = hba->desc_size.pwr_desc;
3080                 break;
3081         case QUERY_DESC_IDN_GEOMETRY:
3082                 *desc_len = hba->desc_size.geom_desc;
3083                 break;
3084         case QUERY_DESC_IDN_CONFIGURATION:
3085                 *desc_len = hba->desc_size.conf_desc;
3086                 break;
3087         case QUERY_DESC_IDN_UNIT:
3088                 *desc_len = hba->desc_size.unit_desc;
3089                 break;
3090         case QUERY_DESC_IDN_INTERCONNECT:
3091                 *desc_len = hba->desc_size.interc_desc;
3092                 break;
3093         case QUERY_DESC_IDN_STRING:
3094                 *desc_len = QUERY_DESC_MAX_SIZE;
3095                 break;
3096         case QUERY_DESC_IDN_HEALTH:
3097                 *desc_len = hba->desc_size.hlth_desc;
3098                 break;
3099         case QUERY_DESC_IDN_RFU_0:
3100         case QUERY_DESC_IDN_RFU_1:
3101                 *desc_len = 0;
3102                 break;
3103         default:
3104                 *desc_len = 0;
3105                 return -EINVAL;
3106         }
3107         return 0;
3108 }
3109 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3110
3111 /**
3112  * ufshcd_read_desc_param - read the specified descriptor parameter
3113  * @hba: Pointer to adapter instance
3114  * @desc_id: descriptor idn value
3115  * @desc_index: descriptor index
3116  * @param_offset: offset of the parameter to read
3117  * @param_read_buf: pointer to buffer where parameter would be read
3118  * @param_size: sizeof(param_read_buf)
3119  *
3120  * Return 0 in case of success, non-zero otherwise
3121  */
3122 int ufshcd_read_desc_param(struct ufs_hba *hba,
3123                            enum desc_idn desc_id,
3124                            int desc_index,
3125                            u8 param_offset,
3126                            u8 *param_read_buf,
3127                            u8 param_size)
3128 {
3129         int ret;
3130         u8 *desc_buf;
3131         int buff_len;
3132         bool is_kmalloc = true;
3133
3134         /* Safety check */
3135         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3136                 return -EINVAL;
3137
3138         /* Get the max length of descriptor from structure filled up at probe
3139          * time.
3140          */
3141         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3142
3143         /* Sanity checks */
3144         if (ret || !buff_len) {
3145                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3146                         __func__);
3147                 return ret;
3148         }
3149
3150         /* Check whether we need temp memory */
3151         if (param_offset != 0 || param_size < buff_len) {
3152                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3153                 if (!desc_buf)
3154                         return -ENOMEM;
3155         } else {
3156                 desc_buf = param_read_buf;
3157                 is_kmalloc = false;
3158         }
3159
3160         /* Request for full descriptor */
3161         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3162                                         desc_id, desc_index, 0,
3163                                         desc_buf, &buff_len);
3164
3165         if (ret) {
3166                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3167                         __func__, desc_id, desc_index, param_offset, ret);
3168                 goto out;
3169         }
3170
3171         /* Sanity check */
3172         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3173                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3174                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3175                 ret = -EINVAL;
3176                 goto out;
3177         }
3178
3179         /* Check wherher we will not copy more data, than available */
3180         if (is_kmalloc && param_size > buff_len)
3181                 param_size = buff_len;
3182
3183         if (is_kmalloc)
3184                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3185 out:
3186         if (is_kmalloc)
3187                 kfree(desc_buf);
3188         return ret;
3189 }
3190
3191 static inline int ufshcd_read_desc(struct ufs_hba *hba,
3192                                    enum desc_idn desc_id,
3193                                    int desc_index,
3194                                    u8 *buf,
3195                                    u32 size)
3196 {
3197         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3198 }
3199
3200 static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3201                                          u8 *buf,
3202                                          u32 size)
3203 {
3204         return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
3205 }
3206
3207 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
3208 {
3209         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3210 }
3211
3212 /**
3213  * ufshcd_read_string_desc - read string descriptor
3214  * @hba: pointer to adapter instance
3215  * @desc_index: descriptor index
3216  * @buf: pointer to buffer where descriptor would be read
3217  * @size: size of buf
3218  * @ascii: if true convert from unicode to ascii characters
3219  *
3220  * Return 0 in case of success, non-zero otherwise
3221  */
3222 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
3223                             u8 *buf, u32 size, bool ascii)
3224 {
3225         int err = 0;
3226
3227         err = ufshcd_read_desc(hba,
3228                                 QUERY_DESC_IDN_STRING, desc_index, buf, size);
3229
3230         if (err) {
3231                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
3232                         __func__, QUERY_REQ_RETRIES, err);
3233                 goto out;
3234         }
3235
3236         if (ascii) {
3237                 int desc_len;
3238                 int ascii_len;
3239                 int i;
3240                 char *buff_ascii;
3241
3242                 desc_len = buf[0];
3243                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3244                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3245                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
3246                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
3247                                         __func__);
3248                         err = -ENOMEM;
3249                         goto out;
3250                 }
3251
3252                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
3253                 if (!buff_ascii) {
3254                         err = -ENOMEM;
3255                         goto out;
3256                 }
3257
3258                 /*
3259                  * the descriptor contains string in UTF16 format
3260                  * we need to convert to utf-8 so it can be displayed
3261                  */
3262                 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
3263                                 desc_len - QUERY_DESC_HDR_SIZE,
3264                                 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
3265
3266                 /* replace non-printable or non-ASCII characters with spaces */
3267                 for (i = 0; i < ascii_len; i++)
3268                         ufshcd_remove_non_printable(&buff_ascii[i]);
3269
3270                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
3271                                 size - QUERY_DESC_HDR_SIZE);
3272                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
3273                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
3274                 kfree(buff_ascii);