1
2
3
4
5
6
7
8
9
10
11
12#include <linux/async.h>
13#include <linux/devfreq.h>
14#include <linux/nls.h>
15#include <linux/of.h>
16#include <linux/bitfield.h>
17#include <linux/blk-pm.h>
18#include <linux/blkdev.h>
19#include "ufshcd.h"
20#include "ufs_quirks.h"
21#include "unipro.h"
22#include "ufs-sysfs.h"
23#include "ufs_bsg.h"
24#include "ufshcd-crypto.h"
25#include <asm/unaligned.h>
26#include <linux/blkdev.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/ufs.h>
30
31#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
32 UTP_TASK_REQ_COMPL |\
33 UFSHCD_ERROR_MASK)
34
35#define UIC_CMD_TIMEOUT 500
36
37
38#define NOP_OUT_RETRIES 10
39
40#define NOP_OUT_TIMEOUT 50
41
42
43#define QUERY_REQ_RETRIES 3
44
45#define QUERY_REQ_TIMEOUT 1500
46
47
48#define TM_CMD_TIMEOUT 100
49
50
51#define UFS_UIC_COMMAND_RETRIES 3
52
53
54#define DME_LINKSTARTUP_RETRIES 3
55
56
57#define UIC_HIBERN8_ENTER_RETRIES 3
58
59
60#define MAX_HOST_RESET_RETRIES 5
61
62
63#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
64
65
66#define INT_AGGR_DEF_TO 0x02
67
68
69#define RPM_AUTOSUSPEND_DELAY_MS 2000
70
71
72#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
73
74
75#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF
76
77
78#define FDEVICEINIT_COMPL_TIMEOUT 1500
79
80#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
81 ({ \
82 int _ret; \
83 if (_on) \
84 _ret = ufshcd_enable_vreg(_dev, _vreg); \
85 else \
86 _ret = ufshcd_disable_vreg(_dev, _vreg); \
87 _ret; \
88 })
89
90#define ufshcd_hex_dump(prefix_str, buf, len) do { \
91 size_t __len = (len); \
92 print_hex_dump(KERN_ERR, prefix_str, \
93 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
94 16, 4, buf, __len, false); \
95} while (0)
96
97int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
98 const char *prefix)
99{
100 u32 *regs;
101 size_t pos;
102
103 if (offset % 4 != 0 || len % 4 != 0)
104 return -EINVAL;
105
106 regs = kzalloc(len, GFP_ATOMIC);
107 if (!regs)
108 return -ENOMEM;
109
110 for (pos = 0; pos < len; pos += 4)
111 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
112
113 ufshcd_hex_dump(prefix, regs, len);
114 kfree(regs);
115
116 return 0;
117}
118EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
119
120enum {
121 UFSHCD_MAX_CHANNEL = 0,
122 UFSHCD_MAX_ID = 1,
123 UFSHCD_CMD_PER_LUN = 32,
124 UFSHCD_CAN_QUEUE = 32,
125};
126
127
128enum {
129 UFSHCD_STATE_RESET,
130 UFSHCD_STATE_ERROR,
131 UFSHCD_STATE_OPERATIONAL,
132 UFSHCD_STATE_EH_SCHEDULED_FATAL,
133 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
134};
135
136
137enum {
138 UFSHCD_EH_IN_PROGRESS = (1 << 0),
139};
140
141
142enum {
143 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0),
144 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1),
145 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2),
146 UFSHCD_UIC_NL_ERROR = (1 << 3),
147 UFSHCD_UIC_TL_ERROR = (1 << 4),
148 UFSHCD_UIC_DME_ERROR = (1 << 5),
149 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6),
150};
151
152#define ufshcd_set_eh_in_progress(h) \
153 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
154#define ufshcd_eh_in_progress(h) \
155 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
156#define ufshcd_clear_eh_in_progress(h) \
157 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
158
159struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
160 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
161 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
162 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
163 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
164 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
165 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
166};
167
168static inline enum ufs_dev_pwr_mode
169ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
170{
171 return ufs_pm_lvl_states[lvl].dev_state;
172}
173
174static inline enum uic_link_state
175ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
176{
177 return ufs_pm_lvl_states[lvl].link_state;
178}
179
180static inline enum ufs_pm_level
181ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
182 enum uic_link_state link_state)
183{
184 enum ufs_pm_level lvl;
185
186 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
187 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
188 (ufs_pm_lvl_states[lvl].link_state == link_state))
189 return lvl;
190 }
191
192
193 return UFS_PM_LVL_0;
194}
195
196static struct ufs_dev_fix ufs_fixups[] = {
197
198 UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
199 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
200 UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
201 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
202 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
203 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
204 UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
205 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
206 UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" ,
207 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
208 UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
209 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
210 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
211 UFS_DEVICE_QUIRK_PA_TACTIVATE),
212 UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
213 UFS_DEVICE_QUIRK_PA_TACTIVATE),
214 END_FIX
215};
216
217static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
218static void ufshcd_async_scan(void *data, async_cookie_t cookie);
219static int ufshcd_reset_and_restore(struct ufs_hba *hba);
220static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
221static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
222static void ufshcd_hba_exit(struct ufs_hba *hba);
223static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
224static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
225 bool skip_ref_clk);
226static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
227static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
228static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
229static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
230static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
231static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
232static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
233static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
234static irqreturn_t ufshcd_intr(int irq, void *__hba);
235static int ufshcd_change_power_mode(struct ufs_hba *hba,
236 struct ufs_pa_layer_attr *pwr_mode);
237static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
238static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
239static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
240static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
241 struct ufs_vreg *vreg);
242static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
243static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
244static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
245static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
246static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
247static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
248
249static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
250{
251 return tag >= 0 && tag < hba->nutrs;
252}
253
254static inline void ufshcd_enable_irq(struct ufs_hba *hba)
255{
256 if (!hba->is_irq_enabled) {
257 enable_irq(hba->irq);
258 hba->is_irq_enabled = true;
259 }
260}
261
262static inline void ufshcd_disable_irq(struct ufs_hba *hba)
263{
264 if (hba->is_irq_enabled) {
265 disable_irq(hba->irq);
266 hba->is_irq_enabled = false;
267 }
268}
269
270static inline void ufshcd_wb_config(struct ufs_hba *hba)
271{
272 int ret;
273
274 if (!ufshcd_is_wb_allowed(hba))
275 return;
276
277 ret = ufshcd_wb_ctrl(hba, true);
278 if (ret)
279 dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
280 else
281 dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
282 ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
283 if (ret)
284 dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
285 __func__, ret);
286 ufshcd_wb_toggle_flush(hba, true);
287}
288
289static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
290{
291 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
292 scsi_unblock_requests(hba->host);
293}
294
295static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
296{
297 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
298 scsi_block_requests(hba->host);
299}
300
301static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
302 const char *str)
303{
304 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
305
306 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
307}
308
309static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
310 const char *str)
311{
312 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
313
314 trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
315}
316
317static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
318 const char *str)
319{
320 int off = (int)tag - hba->nutrs;
321 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
322
323 trace_ufshcd_upiu(dev_name(hba->dev), str, &descp->req_header,
324 &descp->input_param1);
325}
326
327static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
328 struct uic_command *ucmd,
329 const char *str)
330{
331 u32 cmd;
332
333 if (!trace_ufshcd_uic_command_enabled())
334 return;
335
336 if (!strcmp(str, "send"))
337 cmd = ucmd->command;
338 else
339 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
340
341 trace_ufshcd_uic_command(dev_name(hba->dev), str, cmd,
342 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
343 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
344 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
345}
346
347static void ufshcd_add_command_trace(struct ufs_hba *hba,
348 unsigned int tag, const char *str)
349{
350 sector_t lba = -1;
351 u8 opcode = 0;
352 u32 intr, doorbell;
353 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
354 struct scsi_cmnd *cmd = lrbp->cmd;
355 int transfer_len = -1;
356
357 if (!trace_ufshcd_command_enabled()) {
358
359 if (cmd)
360 ufshcd_add_cmd_upiu_trace(hba, tag, str);
361 return;
362 }
363
364 if (cmd) {
365
366 ufshcd_add_cmd_upiu_trace(hba, tag, str);
367 opcode = cmd->cmnd[0];
368 if ((opcode == READ_10) || (opcode == WRITE_10)) {
369
370
371
372
373 if (cmd->request && cmd->request->bio)
374 lba = cmd->request->bio->bi_iter.bi_sector;
375 transfer_len = be32_to_cpu(
376 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
377 }
378 }
379
380 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
381 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
382 trace_ufshcd_command(dev_name(hba->dev), str, tag,
383 doorbell, transfer_len, intr, lba, opcode);
384}
385
386static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
387{
388 struct ufs_clk_info *clki;
389 struct list_head *head = &hba->clk_list_head;
390
391 if (list_empty(head))
392 return;
393
394 list_for_each_entry(clki, head, list) {
395 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
396 clki->max_freq)
397 dev_err(hba->dev, "clk: %s, rate: %u\n",
398 clki->name, clki->curr_freq);
399 }
400}
401
402static void ufshcd_print_err_hist(struct ufs_hba *hba,
403 struct ufs_err_reg_hist *err_hist,
404 char *err_name)
405{
406 int i;
407 bool found = false;
408
409 for (i = 0; i < UFS_ERR_REG_HIST_LENGTH; i++) {
410 int p = (i + err_hist->pos) % UFS_ERR_REG_HIST_LENGTH;
411
412 if (err_hist->tstamp[p] == 0)
413 continue;
414 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
415 err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
416 found = true;
417 }
418
419 if (!found)
420 dev_err(hba->dev, "No record of %s\n", err_name);
421}
422
423static void ufshcd_print_host_regs(struct ufs_hba *hba)
424{
425 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
426
427 ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
428 ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
429 ufshcd_print_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
430 ufshcd_print_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
431 ufshcd_print_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
432 ufshcd_print_err_hist(hba, &hba->ufs_stats.auto_hibern8_err,
433 "auto_hibern8_err");
434 ufshcd_print_err_hist(hba, &hba->ufs_stats.fatal_err, "fatal_err");
435 ufshcd_print_err_hist(hba, &hba->ufs_stats.link_startup_err,
436 "link_startup_fail");
437 ufshcd_print_err_hist(hba, &hba->ufs_stats.resume_err, "resume_fail");
438 ufshcd_print_err_hist(hba, &hba->ufs_stats.suspend_err,
439 "suspend_fail");
440 ufshcd_print_err_hist(hba, &hba->ufs_stats.dev_reset, "dev_reset");
441 ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
442 ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
443
444 ufshcd_vops_dbg_register_dump(hba);
445}
446
447static
448void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
449{
450 struct ufshcd_lrb *lrbp;
451 int prdt_length;
452 int tag;
453
454 for_each_set_bit(tag, &bitmap, hba->nutrs) {
455 lrbp = &hba->lrb[tag];
456
457 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
458 tag, ktime_to_us(lrbp->issue_time_stamp));
459 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
460 tag, ktime_to_us(lrbp->compl_time_stamp));
461 dev_err(hba->dev,
462 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
463 tag, (u64)lrbp->utrd_dma_addr);
464
465 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
466 sizeof(struct utp_transfer_req_desc));
467 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
468 (u64)lrbp->ucd_req_dma_addr);
469 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
470 sizeof(struct utp_upiu_req));
471 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
472 (u64)lrbp->ucd_rsp_dma_addr);
473 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
474 sizeof(struct utp_upiu_rsp));
475
476 prdt_length = le16_to_cpu(
477 lrbp->utr_descriptor_ptr->prd_table_length);
478 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
479 prdt_length /= sizeof(struct ufshcd_sg_entry);
480
481 dev_err(hba->dev,
482 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
483 tag, prdt_length,
484 (u64)lrbp->ucd_prdt_dma_addr);
485
486 if (pr_prdt)
487 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
488 sizeof(struct ufshcd_sg_entry) * prdt_length);
489 }
490}
491
492static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
493{
494 int tag;
495
496 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
497 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
498
499 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
500 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
501 }
502}
503
504static void ufshcd_print_host_state(struct ufs_hba *hba)
505{
506 struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
507
508 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
509 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
510 hba->outstanding_reqs, hba->outstanding_tasks);
511 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
512 hba->saved_err, hba->saved_uic_err);
513 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
514 hba->curr_dev_pwr_mode, hba->uic_link_state);
515 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
516 hba->pm_op_in_progress, hba->is_sys_suspended);
517 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
518 hba->auto_bkops_enabled, hba->host->host_self_blocked);
519 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
520 dev_err(hba->dev,
521 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
522 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
523 hba->ufs_stats.hibern8_exit_cnt);
524 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
525 ktime_to_us(hba->ufs_stats.last_intr_ts),
526 hba->ufs_stats.last_intr_status);
527 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
528 hba->eh_flags, hba->req_abort_count);
529 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
530 hba->ufs_version, hba->capabilities, hba->caps);
531 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
532 hba->dev_quirks);
533 if (sdev_ufs)
534 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
535 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
536
537 ufshcd_print_clk_freqs(hba);
538}
539
540
541
542
543
544
545static void ufshcd_print_pwr_info(struct ufs_hba *hba)
546{
547 static const char * const names[] = {
548 "INVALID MODE",
549 "FAST MODE",
550 "SLOW_MODE",
551 "INVALID MODE",
552 "FASTAUTO_MODE",
553 "SLOWAUTO_MODE",
554 "INVALID MODE",
555 };
556
557 dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
558 __func__,
559 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
560 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
561 names[hba->pwr_info.pwr_rx],
562 names[hba->pwr_info.pwr_tx],
563 hba->pwr_info.hs_rate);
564}
565
566void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
567{
568 if (!us)
569 return;
570
571 if (us < 10)
572 udelay(us);
573 else
574 usleep_range(us, us + tolerance);
575}
576EXPORT_SYMBOL_GPL(ufshcd_delay_us);
577
578
579
580
581
582
583
584
585
586
587
588
589
590int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
591 u32 val, unsigned long interval_us,
592 unsigned long timeout_ms)
593{
594 int err = 0;
595 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
596
597
598 val = val & mask;
599
600 while ((ufshcd_readl(hba, reg) & mask) != val) {
601 usleep_range(interval_us, interval_us + 50);
602 if (time_after(jiffies, timeout)) {
603 if ((ufshcd_readl(hba, reg) & mask) != val)
604 err = -ETIMEDOUT;
605 break;
606 }
607 }
608
609 return err;
610}
611
612
613
614
615
616
617
618static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
619{
620 u32 intr_mask = 0;
621
622 switch (hba->ufs_version) {
623 case UFSHCI_VERSION_10:
624 intr_mask = INTERRUPT_MASK_ALL_VER_10;
625 break;
626 case UFSHCI_VERSION_11:
627 case UFSHCI_VERSION_20:
628 intr_mask = INTERRUPT_MASK_ALL_VER_11;
629 break;
630 case UFSHCI_VERSION_21:
631 default:
632 intr_mask = INTERRUPT_MASK_ALL_VER_21;
633 break;
634 }
635
636 return intr_mask;
637}
638
639
640
641
642
643
644
645static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
646{
647 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
648 return ufshcd_vops_get_ufs_hci_version(hba);
649
650 return ufshcd_readl(hba, REG_UFS_VERSION);
651}
652
653
654
655
656
657
658
659
660static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
661{
662 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
663 DEVICE_PRESENT) ? true : false;
664}
665
666
667
668
669
670
671
672
673static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
674{
675 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
676}
677
678
679
680
681
682
683static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
684{
685 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
686 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
687 else
688 ufshcd_writel(hba, ~(1 << pos),
689 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
690}
691
692
693
694
695
696
697static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
698{
699 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
700 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
701 else
702 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
703}
704
705
706
707
708
709
710static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
711{
712 __clear_bit(tag, &hba->outstanding_reqs);
713}
714
715
716
717
718
719
720
721static inline int ufshcd_get_lists_status(u32 reg)
722{
723 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
724}
725
726
727
728
729
730
731
732
733static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
734{
735 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
736 MASK_UIC_COMMAND_RESULT;
737}
738
739
740
741
742
743
744
745
746static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
747{
748 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
749}
750
751
752
753
754
755static inline int
756ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
757{
758 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
759}
760
761
762
763
764
765
766
767
768static inline int
769ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
770{
771 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
772}
773
774
775
776
777
778
779
780
781static inline unsigned int
782ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
783{
784 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
785 MASK_RSP_UPIU_DATA_SEG_LEN;
786}
787
788
789
790
791
792
793
794
795
796
797static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
798{
799 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
800 MASK_RSP_EXCEPTION_EVENT ? true : false;
801}
802
803
804
805
806
807static inline void
808ufshcd_reset_intr_aggr(struct ufs_hba *hba)
809{
810 ufshcd_writel(hba, INT_AGGR_ENABLE |
811 INT_AGGR_COUNTER_AND_TIMER_RESET,
812 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
813}
814
815
816
817
818
819
820
821static inline void
822ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
823{
824 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
825 INT_AGGR_COUNTER_THLD_VAL(cnt) |
826 INT_AGGR_TIMEOUT_VAL(tmout),
827 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
828}
829
830
831
832
833
834static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
835{
836 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
837}
838
839
840
841
842
843
844
845static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
846{
847 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
848 REG_UTP_TASK_REQ_LIST_RUN_STOP);
849 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
850 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
851}
852
853
854
855
856
857static inline void ufshcd_hba_start(struct ufs_hba *hba)
858{
859 u32 val = CONTROLLER_ENABLE;
860
861 if (ufshcd_crypto_enable(hba))
862 val |= CRYPTO_GENERAL_ENABLE;
863
864 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
865}
866
867
868
869
870
871
872
873static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
874{
875 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
876 ? false : true;
877}
878
879u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
880{
881
882 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
883 (hba->ufs_version == UFSHCI_VERSION_11))
884 return UFS_UNIPRO_VER_1_41;
885 else
886 return UFS_UNIPRO_VER_1_6;
887}
888EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
889
890static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
891{
892
893
894
895
896
897
898
899
900
901 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
902 return true;
903 else
904 return false;
905}
906
907
908
909
910
911
912
913
914
915static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
916{
917 int ret = 0;
918 struct ufs_clk_info *clki;
919 struct list_head *head = &hba->clk_list_head;
920
921 if (list_empty(head))
922 goto out;
923
924 list_for_each_entry(clki, head, list) {
925 if (!IS_ERR_OR_NULL(clki->clk)) {
926 if (scale_up && clki->max_freq) {
927 if (clki->curr_freq == clki->max_freq)
928 continue;
929
930 ret = clk_set_rate(clki->clk, clki->max_freq);
931 if (ret) {
932 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
933 __func__, clki->name,
934 clki->max_freq, ret);
935 break;
936 }
937 trace_ufshcd_clk_scaling(dev_name(hba->dev),
938 "scaled up", clki->name,
939 clki->curr_freq,
940 clki->max_freq);
941
942 clki->curr_freq = clki->max_freq;
943
944 } else if (!scale_up && clki->min_freq) {
945 if (clki->curr_freq == clki->min_freq)
946 continue;
947
948 ret = clk_set_rate(clki->clk, clki->min_freq);
949 if (ret) {
950 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
951 __func__, clki->name,
952 clki->min_freq, ret);
953 break;
954 }
955 trace_ufshcd_clk_scaling(dev_name(hba->dev),
956 "scaled down", clki->name,
957 clki->curr_freq,
958 clki->min_freq);
959 clki->curr_freq = clki->min_freq;
960 }
961 }
962 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
963 clki->name, clk_get_rate(clki->clk));
964 }
965
966out:
967 return ret;
968}
969
970
971
972
973
974
975
976
977
978static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
979{
980 int ret = 0;
981 ktime_t start = ktime_get();
982
983 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
984 if (ret)
985 goto out;
986
987 ret = ufshcd_set_clk_freq(hba, scale_up);
988 if (ret)
989 goto out;
990
991 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
992 if (ret)
993 ufshcd_set_clk_freq(hba, !scale_up);
994
995out:
996 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
997 (scale_up ? "up" : "down"),
998 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
999 return ret;
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1010 bool scale_up)
1011{
1012 struct ufs_clk_info *clki;
1013 struct list_head *head = &hba->clk_list_head;
1014
1015 if (list_empty(head))
1016 return false;
1017
1018 list_for_each_entry(clki, head, list) {
1019 if (!IS_ERR_OR_NULL(clki->clk)) {
1020 if (scale_up && clki->max_freq) {
1021 if (clki->curr_freq == clki->max_freq)
1022 continue;
1023 return true;
1024 } else if (!scale_up && clki->min_freq) {
1025 if (clki->curr_freq == clki->min_freq)
1026 continue;
1027 return true;
1028 }
1029 }
1030 }
1031
1032 return false;
1033}
1034
1035static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1036 u64 wait_timeout_us)
1037{
1038 unsigned long flags;
1039 int ret = 0;
1040 u32 tm_doorbell;
1041 u32 tr_doorbell;
1042 bool timeout = false, do_last_check = false;
1043 ktime_t start;
1044
1045 ufshcd_hold(hba, false);
1046 spin_lock_irqsave(hba->host->host_lock, flags);
1047
1048
1049
1050
1051 start = ktime_get();
1052 do {
1053 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1054 ret = -EBUSY;
1055 goto out;
1056 }
1057
1058 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1059 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1060 if (!tm_doorbell && !tr_doorbell) {
1061 timeout = false;
1062 break;
1063 } else if (do_last_check) {
1064 break;
1065 }
1066
1067 spin_unlock_irqrestore(hba->host->host_lock, flags);
1068 schedule();
1069 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1070 wait_timeout_us) {
1071 timeout = true;
1072
1073
1074
1075
1076
1077 do_last_check = true;
1078 }
1079 spin_lock_irqsave(hba->host->host_lock, flags);
1080 } while (tm_doorbell || tr_doorbell);
1081
1082 if (timeout) {
1083 dev_err(hba->dev,
1084 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1085 __func__, tm_doorbell, tr_doorbell);
1086 ret = -EBUSY;
1087 }
1088out:
1089 spin_unlock_irqrestore(hba->host->host_lock, flags);
1090 ufshcd_release(hba);
1091 return ret;
1092}
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1104{
1105 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1106 int ret = 0;
1107 struct ufs_pa_layer_attr new_pwr_info;
1108
1109 if (scale_up) {
1110 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1111 sizeof(struct ufs_pa_layer_attr));
1112 } else {
1113 memcpy(&new_pwr_info, &hba->pwr_info,
1114 sizeof(struct ufs_pa_layer_attr));
1115
1116 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1117 || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1118
1119 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1120 &hba->pwr_info,
1121 sizeof(struct ufs_pa_layer_attr));
1122
1123
1124 new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1125 new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1126 }
1127 }
1128
1129
1130 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1131 if (ret)
1132 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1133 __func__, ret,
1134 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1135 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1136
1137 return ret;
1138}
1139
1140static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1141{
1142 #define DOORBELL_CLR_TOUT_US (1000 * 1000)
1143 int ret = 0;
1144
1145
1146
1147
1148 ufshcd_scsi_block_requests(hba);
1149 down_write(&hba->clk_scaling_lock);
1150 if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1151 ret = -EBUSY;
1152 up_write(&hba->clk_scaling_lock);
1153 ufshcd_scsi_unblock_requests(hba);
1154 }
1155
1156 return ret;
1157}
1158
1159static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1160{
1161 up_write(&hba->clk_scaling_lock);
1162 ufshcd_scsi_unblock_requests(hba);
1163}
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1175{
1176 int ret = 0;
1177
1178
1179 ufshcd_hold(hba, false);
1180
1181 ret = ufshcd_clock_scaling_prepare(hba);
1182 if (ret)
1183 goto out;
1184
1185
1186 if (!scale_up) {
1187 ret = ufshcd_scale_gear(hba, false);
1188 if (ret)
1189 goto out_unprepare;
1190 }
1191
1192 ret = ufshcd_scale_clks(hba, scale_up);
1193 if (ret) {
1194 if (!scale_up)
1195 ufshcd_scale_gear(hba, true);
1196 goto out_unprepare;
1197 }
1198
1199
1200 if (scale_up) {
1201 ret = ufshcd_scale_gear(hba, true);
1202 if (ret) {
1203 ufshcd_scale_clks(hba, false);
1204 goto out_unprepare;
1205 }
1206 }
1207
1208
1209 up_write(&hba->clk_scaling_lock);
1210 ufshcd_wb_ctrl(hba, scale_up);
1211 down_write(&hba->clk_scaling_lock);
1212
1213out_unprepare:
1214 ufshcd_clock_scaling_unprepare(hba);
1215out:
1216 ufshcd_release(hba);
1217 return ret;
1218}
1219
1220static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1221{
1222 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1223 clk_scaling.suspend_work);
1224 unsigned long irq_flags;
1225
1226 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1227 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1228 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1229 return;
1230 }
1231 hba->clk_scaling.is_suspended = true;
1232 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1233
1234 __ufshcd_suspend_clkscaling(hba);
1235}
1236
1237static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1238{
1239 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1240 clk_scaling.resume_work);
1241 unsigned long irq_flags;
1242
1243 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1244 if (!hba->clk_scaling.is_suspended) {
1245 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1246 return;
1247 }
1248 hba->clk_scaling.is_suspended = false;
1249 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1250
1251 devfreq_resume_device(hba->devfreq);
1252}
1253
1254static int ufshcd_devfreq_target(struct device *dev,
1255 unsigned long *freq, u32 flags)
1256{
1257 int ret = 0;
1258 struct ufs_hba *hba = dev_get_drvdata(dev);
1259 ktime_t start;
1260 bool scale_up, sched_clk_scaling_suspend_work = false;
1261 struct list_head *clk_list = &hba->clk_list_head;
1262 struct ufs_clk_info *clki;
1263 unsigned long irq_flags;
1264
1265 if (!ufshcd_is_clkscaling_supported(hba))
1266 return -EINVAL;
1267
1268 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1269
1270 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1271 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1272 if (ufshcd_eh_in_progress(hba)) {
1273 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1274 return 0;
1275 }
1276
1277 if (!hba->clk_scaling.active_reqs)
1278 sched_clk_scaling_suspend_work = true;
1279
1280 if (list_empty(clk_list)) {
1281 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1282 goto out;
1283 }
1284
1285
1286 scale_up = (*freq == clki->max_freq) ? true : false;
1287 if (!scale_up)
1288 *freq = clki->min_freq;
1289
1290 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1291 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1292 ret = 0;
1293 goto out;
1294 }
1295 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1296
1297 pm_runtime_get_noresume(hba->dev);
1298 if (!pm_runtime_active(hba->dev)) {
1299 pm_runtime_put_noidle(hba->dev);
1300 ret = -EAGAIN;
1301 goto out;
1302 }
1303 start = ktime_get();
1304 ret = ufshcd_devfreq_scale(hba, scale_up);
1305 pm_runtime_put(hba->dev);
1306
1307 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1308 (scale_up ? "up" : "down"),
1309 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1310
1311out:
1312 if (sched_clk_scaling_suspend_work)
1313 queue_work(hba->clk_scaling.workq,
1314 &hba->clk_scaling.suspend_work);
1315
1316 return ret;
1317}
1318
1319static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1320{
1321 int *busy = priv;
1322
1323 WARN_ON_ONCE(reserved);
1324 (*busy)++;
1325 return false;
1326}
1327
1328
1329static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1330{
1331 struct request_queue *q = hba->cmd_queue;
1332 int busy = 0;
1333
1334 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1335 return busy;
1336}
1337
1338static int ufshcd_devfreq_get_dev_status(struct device *dev,
1339 struct devfreq_dev_status *stat)
1340{
1341 struct ufs_hba *hba = dev_get_drvdata(dev);
1342 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1343 unsigned long flags;
1344 struct list_head *clk_list = &hba->clk_list_head;
1345 struct ufs_clk_info *clki;
1346 ktime_t curr_t;
1347
1348 if (!ufshcd_is_clkscaling_supported(hba))
1349 return -EINVAL;
1350
1351 memset(stat, 0, sizeof(*stat));
1352
1353 spin_lock_irqsave(hba->host->host_lock, flags);
1354 curr_t = ktime_get();
1355 if (!scaling->window_start_t)
1356 goto start_window;
1357
1358 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1359
1360
1361
1362
1363
1364 stat->current_frequency = clki->curr_freq;
1365 if (scaling->is_busy_started)
1366 scaling->tot_busy_t += ktime_us_delta(curr_t,
1367 scaling->busy_start_t);
1368
1369 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1370 stat->busy_time = scaling->tot_busy_t;
1371start_window:
1372 scaling->window_start_t = curr_t;
1373 scaling->tot_busy_t = 0;
1374
1375 if (hba->outstanding_reqs) {
1376 scaling->busy_start_t = curr_t;
1377 scaling->is_busy_started = true;
1378 } else {
1379 scaling->busy_start_t = 0;
1380 scaling->is_busy_started = false;
1381 }
1382 spin_unlock_irqrestore(hba->host->host_lock, flags);
1383 return 0;
1384}
1385
1386static int ufshcd_devfreq_init(struct ufs_hba *hba)
1387{
1388 struct list_head *clk_list = &hba->clk_list_head;
1389 struct ufs_clk_info *clki;
1390 struct devfreq *devfreq;
1391 int ret;
1392
1393
1394 if (list_empty(clk_list))
1395 return 0;
1396
1397 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1398 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1399 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1400
1401 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1402 &hba->vps->ondemand_data);
1403 devfreq = devfreq_add_device(hba->dev,
1404 &hba->vps->devfreq_profile,
1405 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1406 &hba->vps->ondemand_data);
1407 if (IS_ERR(devfreq)) {
1408 ret = PTR_ERR(devfreq);
1409 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1410
1411 dev_pm_opp_remove(hba->dev, clki->min_freq);
1412 dev_pm_opp_remove(hba->dev, clki->max_freq);
1413 return ret;
1414 }
1415
1416 hba->devfreq = devfreq;
1417
1418 return 0;
1419}
1420
1421static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1422{
1423 struct list_head *clk_list = &hba->clk_list_head;
1424 struct ufs_clk_info *clki;
1425
1426 if (!hba->devfreq)
1427 return;
1428
1429 devfreq_remove_device(hba->devfreq);
1430 hba->devfreq = NULL;
1431
1432 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1433 dev_pm_opp_remove(hba->dev, clki->min_freq);
1434 dev_pm_opp_remove(hba->dev, clki->max_freq);
1435}
1436
1437static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1438{
1439 unsigned long flags;
1440
1441 devfreq_suspend_device(hba->devfreq);
1442 spin_lock_irqsave(hba->host->host_lock, flags);
1443 hba->clk_scaling.window_start_t = 0;
1444 spin_unlock_irqrestore(hba->host->host_lock, flags);
1445}
1446
1447static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1448{
1449 unsigned long flags;
1450 bool suspend = false;
1451
1452 if (!ufshcd_is_clkscaling_supported(hba))
1453 return;
1454
1455 spin_lock_irqsave(hba->host->host_lock, flags);
1456 if (!hba->clk_scaling.is_suspended) {
1457 suspend = true;
1458 hba->clk_scaling.is_suspended = true;
1459 }
1460 spin_unlock_irqrestore(hba->host->host_lock, flags);
1461
1462 if (suspend)
1463 __ufshcd_suspend_clkscaling(hba);
1464}
1465
1466static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1467{
1468 unsigned long flags;
1469 bool resume = false;
1470
1471 if (!ufshcd_is_clkscaling_supported(hba))
1472 return;
1473
1474 spin_lock_irqsave(hba->host->host_lock, flags);
1475 if (hba->clk_scaling.is_suspended) {
1476 resume = true;
1477 hba->clk_scaling.is_suspended = false;
1478 }
1479 spin_unlock_irqrestore(hba->host->host_lock, flags);
1480
1481 if (resume)
1482 devfreq_resume_device(hba->devfreq);
1483}
1484
1485static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1486 struct device_attribute *attr, char *buf)
1487{
1488 struct ufs_hba *hba = dev_get_drvdata(dev);
1489
1490 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1491}
1492
1493static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1494 struct device_attribute *attr, const char *buf, size_t count)
1495{
1496 struct ufs_hba *hba = dev_get_drvdata(dev);
1497 u32 value;
1498 int err;
1499
1500 if (kstrtou32(buf, 0, &value))
1501 return -EINVAL;
1502
1503 value = !!value;
1504 if (value == hba->clk_scaling.is_allowed)
1505 goto out;
1506
1507 pm_runtime_get_sync(hba->dev);
1508 ufshcd_hold(hba, false);
1509
1510 cancel_work_sync(&hba->clk_scaling.suspend_work);
1511 cancel_work_sync(&hba->clk_scaling.resume_work);
1512
1513 hba->clk_scaling.is_allowed = value;
1514
1515 if (value) {
1516 ufshcd_resume_clkscaling(hba);
1517 } else {
1518 ufshcd_suspend_clkscaling(hba);
1519 err = ufshcd_devfreq_scale(hba, true);
1520 if (err)
1521 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1522 __func__, err);
1523 }
1524
1525 ufshcd_release(hba);
1526 pm_runtime_put_sync(hba->dev);
1527out:
1528 return count;
1529}
1530
1531static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1532{
1533 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1534 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1535 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1536 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1537 hba->clk_scaling.enable_attr.attr.mode = 0644;
1538 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1539 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1540}
1541
1542static void ufshcd_ungate_work(struct work_struct *work)
1543{
1544 int ret;
1545 unsigned long flags;
1546 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1547 clk_gating.ungate_work);
1548
1549 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1550
1551 spin_lock_irqsave(hba->host->host_lock, flags);
1552 if (hba->clk_gating.state == CLKS_ON) {
1553 spin_unlock_irqrestore(hba->host->host_lock, flags);
1554 goto unblock_reqs;
1555 }
1556
1557 spin_unlock_irqrestore(hba->host->host_lock, flags);
1558 ufshcd_setup_clocks(hba, true);
1559
1560 ufshcd_enable_irq(hba);
1561
1562
1563 if (ufshcd_can_hibern8_during_gating(hba)) {
1564
1565 hba->clk_gating.is_suspended = true;
1566 if (ufshcd_is_link_hibern8(hba)) {
1567 ret = ufshcd_uic_hibern8_exit(hba);
1568 if (ret)
1569 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1570 __func__, ret);
1571 else
1572 ufshcd_set_link_active(hba);
1573 }
1574 hba->clk_gating.is_suspended = false;
1575 }
1576unblock_reqs:
1577 ufshcd_scsi_unblock_requests(hba);
1578}
1579
1580
1581
1582
1583
1584
1585
1586int ufshcd_hold(struct ufs_hba *hba, bool async)
1587{
1588 int rc = 0;
1589 bool flush_result;
1590 unsigned long flags;
1591
1592 if (!ufshcd_is_clkgating_allowed(hba))
1593 goto out;
1594 spin_lock_irqsave(hba->host->host_lock, flags);
1595 hba->clk_gating.active_reqs++;
1596
1597start:
1598 switch (hba->clk_gating.state) {
1599 case CLKS_ON:
1600
1601
1602
1603
1604
1605
1606
1607
1608 if (ufshcd_can_hibern8_during_gating(hba) &&
1609 ufshcd_is_link_hibern8(hba)) {
1610 if (async) {
1611 rc = -EAGAIN;
1612 hba->clk_gating.active_reqs--;
1613 break;
1614 }
1615 spin_unlock_irqrestore(hba->host->host_lock, flags);
1616 flush_result = flush_work(&hba->clk_gating.ungate_work);
1617 if (hba->clk_gating.is_suspended && !flush_result)
1618 goto out;
1619 spin_lock_irqsave(hba->host->host_lock, flags);
1620 goto start;
1621 }
1622 break;
1623 case REQ_CLKS_OFF:
1624 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1625 hba->clk_gating.state = CLKS_ON;
1626 trace_ufshcd_clk_gating(dev_name(hba->dev),
1627 hba->clk_gating.state);
1628 break;
1629 }
1630
1631
1632
1633
1634
1635 fallthrough;
1636 case CLKS_OFF:
1637 hba->clk_gating.state = REQ_CLKS_ON;
1638 trace_ufshcd_clk_gating(dev_name(hba->dev),
1639 hba->clk_gating.state);
1640 if (queue_work(hba->clk_gating.clk_gating_workq,
1641 &hba->clk_gating.ungate_work))
1642 ufshcd_scsi_block_requests(hba);
1643
1644
1645
1646
1647 fallthrough;
1648 case REQ_CLKS_ON:
1649 if (async) {
1650 rc = -EAGAIN;
1651 hba->clk_gating.active_reqs--;
1652 break;
1653 }
1654
1655 spin_unlock_irqrestore(hba->host->host_lock, flags);
1656 flush_work(&hba->clk_gating.ungate_work);
1657
1658 spin_lock_irqsave(hba->host->host_lock, flags);
1659 goto start;
1660 default:
1661 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1662 __func__, hba->clk_gating.state);
1663 break;
1664 }
1665 spin_unlock_irqrestore(hba->host->host_lock, flags);
1666out:
1667 return rc;
1668}
1669EXPORT_SYMBOL_GPL(ufshcd_hold);
1670
1671static void ufshcd_gate_work(struct work_struct *work)
1672{
1673 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1674 clk_gating.gate_work.work);
1675 unsigned long flags;
1676 int ret;
1677
1678 spin_lock_irqsave(hba->host->host_lock, flags);
1679
1680
1681
1682
1683
1684
1685 if (hba->clk_gating.is_suspended ||
1686 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1687 hba->clk_gating.state = CLKS_ON;
1688 trace_ufshcd_clk_gating(dev_name(hba->dev),
1689 hba->clk_gating.state);
1690 goto rel_lock;
1691 }
1692
1693 if (hba->clk_gating.active_reqs
1694 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1695 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1696 || hba->active_uic_cmd || hba->uic_async_done)
1697 goto rel_lock;
1698
1699 spin_unlock_irqrestore(hba->host->host_lock, flags);
1700
1701
1702 if (ufshcd_can_hibern8_during_gating(hba)) {
1703 ret = ufshcd_uic_hibern8_enter(hba);
1704 if (ret) {
1705 hba->clk_gating.state = CLKS_ON;
1706 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1707 __func__, ret);
1708 trace_ufshcd_clk_gating(dev_name(hba->dev),
1709 hba->clk_gating.state);
1710 goto out;
1711 }
1712 ufshcd_set_link_hibern8(hba);
1713 }
1714
1715 ufshcd_disable_irq(hba);
1716
1717 if (!ufshcd_is_link_active(hba))
1718 ufshcd_setup_clocks(hba, false);
1719 else
1720
1721 __ufshcd_setup_clocks(hba, false, true);
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732 spin_lock_irqsave(hba->host->host_lock, flags);
1733 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1734 hba->clk_gating.state = CLKS_OFF;
1735 trace_ufshcd_clk_gating(dev_name(hba->dev),
1736 hba->clk_gating.state);
1737 }
1738rel_lock:
1739 spin_unlock_irqrestore(hba->host->host_lock, flags);
1740out:
1741 return;
1742}
1743
1744
1745static void __ufshcd_release(struct ufs_hba *hba)
1746{
1747 if (!ufshcd_is_clkgating_allowed(hba))
1748 return;
1749
1750 hba->clk_gating.active_reqs--;
1751
1752 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1753 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1754 ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks ||
1755 hba->active_uic_cmd || hba->uic_async_done)
1756 return;
1757
1758 hba->clk_gating.state = REQ_CLKS_OFF;
1759 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1760 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1761 &hba->clk_gating.gate_work,
1762 msecs_to_jiffies(hba->clk_gating.delay_ms));
1763}
1764
1765void ufshcd_release(struct ufs_hba *hba)
1766{
1767 unsigned long flags;
1768
1769 spin_lock_irqsave(hba->host->host_lock, flags);
1770 __ufshcd_release(hba);
1771 spin_unlock_irqrestore(hba->host->host_lock, flags);
1772}
1773EXPORT_SYMBOL_GPL(ufshcd_release);
1774
1775static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1776 struct device_attribute *attr, char *buf)
1777{
1778 struct ufs_hba *hba = dev_get_drvdata(dev);
1779
1780 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1781}
1782
1783static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1784 struct device_attribute *attr, const char *buf, size_t count)
1785{
1786 struct ufs_hba *hba = dev_get_drvdata(dev);
1787 unsigned long flags, value;
1788
1789 if (kstrtoul(buf, 0, &value))
1790 return -EINVAL;
1791
1792 spin_lock_irqsave(hba->host->host_lock, flags);
1793 hba->clk_gating.delay_ms = value;
1794 spin_unlock_irqrestore(hba->host->host_lock, flags);
1795 return count;
1796}
1797
1798static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1799 struct device_attribute *attr, char *buf)
1800{
1801 struct ufs_hba *hba = dev_get_drvdata(dev);
1802
1803 return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1804}
1805
1806static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1807 struct device_attribute *attr, const char *buf, size_t count)
1808{
1809 struct ufs_hba *hba = dev_get_drvdata(dev);
1810 unsigned long flags;
1811 u32 value;
1812
1813 if (kstrtou32(buf, 0, &value))
1814 return -EINVAL;
1815
1816 value = !!value;
1817 if (value == hba->clk_gating.is_enabled)
1818 goto out;
1819
1820 if (value) {
1821 ufshcd_release(hba);
1822 } else {
1823 spin_lock_irqsave(hba->host->host_lock, flags);
1824 hba->clk_gating.active_reqs++;
1825 spin_unlock_irqrestore(hba->host->host_lock, flags);
1826 }
1827
1828 hba->clk_gating.is_enabled = value;
1829out:
1830 return count;
1831}
1832
1833static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1834{
1835 char wq_name[sizeof("ufs_clkscaling_00")];
1836
1837 if (!ufshcd_is_clkscaling_supported(hba))
1838 return;
1839
1840 INIT_WORK(&hba->clk_scaling.suspend_work,
1841 ufshcd_clk_scaling_suspend_work);
1842 INIT_WORK(&hba->clk_scaling.resume_work,
1843 ufshcd_clk_scaling_resume_work);
1844
1845 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1846 hba->host->host_no);
1847 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1848
1849 ufshcd_clkscaling_init_sysfs(hba);
1850}
1851
1852static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1853{
1854 if (!ufshcd_is_clkscaling_supported(hba))
1855 return;
1856
1857 destroy_workqueue(hba->clk_scaling.workq);
1858 ufshcd_devfreq_remove(hba);
1859}
1860
1861static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1862{
1863 char wq_name[sizeof("ufs_clk_gating_00")];
1864
1865 if (!ufshcd_is_clkgating_allowed(hba))
1866 return;
1867
1868 hba->clk_gating.state = CLKS_ON;
1869
1870 hba->clk_gating.delay_ms = 150;
1871 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1872 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1873
1874 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1875 hba->host->host_no);
1876 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1877 WQ_MEM_RECLAIM);
1878
1879 hba->clk_gating.is_enabled = true;
1880
1881 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1882 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1883 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1884 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1885 hba->clk_gating.delay_attr.attr.mode = 0644;
1886 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1887 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1888
1889 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1890 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1891 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1892 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1893 hba->clk_gating.enable_attr.attr.mode = 0644;
1894 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1895 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1896}
1897
1898static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1899{
1900 if (!ufshcd_is_clkgating_allowed(hba))
1901 return;
1902 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1903 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1904 cancel_work_sync(&hba->clk_gating.ungate_work);
1905 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1906 destroy_workqueue(hba->clk_gating.clk_gating_workq);
1907}
1908
1909
1910static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1911{
1912 bool queue_resume_work = false;
1913 ktime_t curr_t = ktime_get();
1914
1915 if (!ufshcd_is_clkscaling_supported(hba))
1916 return;
1917
1918 if (!hba->clk_scaling.active_reqs++)
1919 queue_resume_work = true;
1920
1921 if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1922 return;
1923
1924 if (queue_resume_work)
1925 queue_work(hba->clk_scaling.workq,
1926 &hba->clk_scaling.resume_work);
1927
1928 if (!hba->clk_scaling.window_start_t) {
1929 hba->clk_scaling.window_start_t = curr_t;
1930 hba->clk_scaling.tot_busy_t = 0;
1931 hba->clk_scaling.is_busy_started = false;
1932 }
1933
1934 if (!hba->clk_scaling.is_busy_started) {
1935 hba->clk_scaling.busy_start_t = curr_t;
1936 hba->clk_scaling.is_busy_started = true;
1937 }
1938}
1939
1940static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1941{
1942 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1943
1944 if (!ufshcd_is_clkscaling_supported(hba))
1945 return;
1946
1947 if (!hba->outstanding_reqs && scaling->is_busy_started) {
1948 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1949 scaling->busy_start_t));
1950 scaling->busy_start_t = 0;
1951 scaling->is_busy_started = false;
1952 }
1953}
1954
1955
1956
1957
1958
1959static inline
1960void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1961{
1962 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
1963
1964 lrbp->issue_time_stamp = ktime_get();
1965 lrbp->compl_time_stamp = ktime_set(0, 0);
1966 ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
1967 ufshcd_add_command_trace(hba, task_tag, "send");
1968 ufshcd_clk_scaling_start_busy(hba);
1969 __set_bit(task_tag, &hba->outstanding_reqs);
1970 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1971
1972 wmb();
1973}
1974
1975
1976
1977
1978
1979static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1980{
1981 int len;
1982 if (lrbp->sense_buffer &&
1983 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1984 int len_to_copy;
1985
1986 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1987 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
1988
1989 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
1990 len_to_copy);
1991 }
1992}
1993
1994
1995
1996
1997
1998
1999
2000static
2001int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2002{
2003 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2004
2005 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2006
2007
2008 if (hba->dev_cmd.query.descriptor &&
2009 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2010 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2011 GENERAL_UPIU_REQUEST_SIZE;
2012 u16 resp_len;
2013 u16 buf_len;
2014
2015
2016 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2017 MASK_QUERY_DATA_SEG_LEN;
2018 buf_len = be16_to_cpu(
2019 hba->dev_cmd.query.request.upiu_req.length);
2020 if (likely(buf_len >= resp_len)) {
2021 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2022 } else {
2023 dev_warn(hba->dev,
2024 "%s: rsp size %d is bigger than buffer size %d",
2025 __func__, resp_len, buf_len);
2026 return -EINVAL;
2027 }
2028 }
2029
2030 return 0;
2031}
2032
2033
2034
2035
2036
2037
2038
2039static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2040{
2041 int err;
2042
2043 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2044
2045
2046 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2047 hba->nutmrs =
2048 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2049
2050
2051 err = ufshcd_hba_init_crypto_capabilities(hba);
2052 if (err)
2053 dev_err(hba->dev, "crypto setup failed\n");
2054
2055 return err;
2056}
2057
2058
2059
2060
2061
2062
2063
2064static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2065{
2066 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2067 return true;
2068 else
2069 return false;
2070}
2071
2072
2073
2074
2075
2076
2077
2078
2079static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2080{
2081 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2082}
2083
2084
2085
2086
2087
2088
2089
2090
2091static inline void
2092ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2093{
2094 WARN_ON(hba->active_uic_cmd);
2095
2096 hba->active_uic_cmd = uic_cmd;
2097
2098
2099 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2100 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2101 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2102
2103 ufshcd_add_uic_command_trace(hba, uic_cmd, "send");
2104
2105
2106 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2107 REG_UIC_COMMAND);
2108}
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118static int
2119ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2120{
2121 int ret;
2122 unsigned long flags;
2123
2124 if (wait_for_completion_timeout(&uic_cmd->done,
2125 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2126 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2127 } else {
2128 ret = -ETIMEDOUT;
2129 dev_err(hba->dev,
2130 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2131 uic_cmd->command, uic_cmd->argument3);
2132
2133 if (!uic_cmd->cmd_active) {
2134 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2135 __func__);
2136 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2137 }
2138 }
2139
2140 spin_lock_irqsave(hba->host->host_lock, flags);
2141 hba->active_uic_cmd = NULL;
2142 spin_unlock_irqrestore(hba->host->host_lock, flags);
2143
2144 return ret;
2145}
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157static int
2158__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2159 bool completion)
2160{
2161 if (!ufshcd_ready_for_uic_cmd(hba)) {
2162 dev_err(hba->dev,
2163 "Controller not ready to accept UIC commands\n");
2164 return -EIO;
2165 }
2166
2167 if (completion)
2168 init_completion(&uic_cmd->done);
2169
2170 uic_cmd->cmd_active = 1;
2171 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2172
2173 return 0;
2174}
2175
2176
2177
2178
2179
2180
2181
2182
2183int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2184{
2185 int ret;
2186 unsigned long flags;
2187
2188 ufshcd_hold(hba, false);
2189 mutex_lock(&hba->uic_cmd_mutex);
2190 ufshcd_add_delay_before_dme_cmd(hba);
2191
2192 spin_lock_irqsave(hba->host->host_lock, flags);
2193 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2194 spin_unlock_irqrestore(hba->host->host_lock, flags);
2195 if (!ret)
2196 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2197
2198 mutex_unlock(&hba->uic_cmd_mutex);
2199
2200 ufshcd_release(hba);
2201 return ret;
2202}
2203
2204
2205
2206
2207
2208
2209
2210
2211static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2212{
2213 struct ufshcd_sg_entry *prd_table;
2214 struct scatterlist *sg;
2215 struct scsi_cmnd *cmd;
2216 int sg_segments;
2217 int i;
2218
2219 cmd = lrbp->cmd;
2220 sg_segments = scsi_dma_map(cmd);
2221 if (sg_segments < 0)
2222 return sg_segments;
2223
2224 if (sg_segments) {
2225
2226 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2227 lrbp->utr_descriptor_ptr->prd_table_length =
2228 cpu_to_le16((sg_segments *
2229 sizeof(struct ufshcd_sg_entry)));
2230 else
2231 lrbp->utr_descriptor_ptr->prd_table_length =
2232 cpu_to_le16((u16) (sg_segments));
2233
2234 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2235
2236 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2237 prd_table[i].size =
2238 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2239 prd_table[i].base_addr =
2240 cpu_to_le32(lower_32_bits(sg->dma_address));
2241 prd_table[i].upper_addr =
2242 cpu_to_le32(upper_32_bits(sg->dma_address));
2243 prd_table[i].reserved = 0;
2244 }
2245 } else {
2246 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2247 }
2248
2249 return 0;
2250}
2251
2252
2253
2254
2255
2256
2257static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2258{
2259 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2260
2261 if (hba->ufs_version == UFSHCI_VERSION_10) {
2262 u32 rw;
2263 rw = set & INTERRUPT_MASK_RW_VER_10;
2264 set = rw | ((set ^ intrs) & intrs);
2265 } else {
2266 set |= intrs;
2267 }
2268
2269 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2270}
2271
2272
2273
2274
2275
2276
2277static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2278{
2279 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2280
2281 if (hba->ufs_version == UFSHCI_VERSION_10) {
2282 u32 rw;
2283 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2284 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2285 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2286
2287 } else {
2288 set &= ~intrs;
2289 }
2290
2291 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2292}
2293
2294
2295
2296
2297
2298
2299
2300
2301static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2302 u8 *upiu_flags, enum dma_data_direction cmd_dir)
2303{
2304 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2305 u32 data_direction;
2306 u32 dword_0;
2307 u32 dword_1 = 0;
2308 u32 dword_3 = 0;
2309
2310 if (cmd_dir == DMA_FROM_DEVICE) {
2311 data_direction = UTP_DEVICE_TO_HOST;
2312 *upiu_flags = UPIU_CMD_FLAGS_READ;
2313 } else if (cmd_dir == DMA_TO_DEVICE) {
2314 data_direction = UTP_HOST_TO_DEVICE;
2315 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2316 } else {
2317 data_direction = UTP_NO_DATA_TRANSFER;
2318 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2319 }
2320
2321 dword_0 = data_direction | (lrbp->command_type
2322 << UPIU_COMMAND_TYPE_OFFSET);
2323 if (lrbp->intr_cmd)
2324 dword_0 |= UTP_REQ_DESC_INT_CMD;
2325
2326
2327 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2328
2329
2330 req_desc->header.dword_0 = cpu_to_le32(dword_0);
2331 req_desc->header.dword_1 = cpu_to_le32(dword_1);
2332
2333
2334
2335
2336
2337 req_desc->header.dword_2 =
2338 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2339 req_desc->header.dword_3 = cpu_to_le32(dword_3);
2340
2341 req_desc->prd_table_length = 0;
2342}
2343
2344
2345
2346
2347
2348
2349
2350static
2351void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2352{
2353 struct scsi_cmnd *cmd = lrbp->cmd;
2354 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2355 unsigned short cdb_len;
2356
2357
2358 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2359 UPIU_TRANSACTION_COMMAND, upiu_flags,
2360 lrbp->lun, lrbp->task_tag);
2361 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2362 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2363
2364
2365 ucd_req_ptr->header.dword_2 = 0;
2366
2367 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2368
2369 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2370 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2371 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2372
2373 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2374}
2375
2376
2377
2378
2379
2380
2381
2382
2383static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2384 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2385{
2386 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2387 struct ufs_query *query = &hba->dev_cmd.query;
2388 u16 len = be16_to_cpu(query->request.upiu_req.length);
2389
2390
2391 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2392 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2393 lrbp->lun, lrbp->task_tag);
2394 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2395 0, query->request.query_func, 0, 0);
2396
2397
2398 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2399 ucd_req_ptr->header.dword_2 =
2400 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2401 else
2402 ucd_req_ptr->header.dword_2 = 0;
2403
2404
2405 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2406 QUERY_OSF_SIZE);
2407
2408
2409 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2410 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2411
2412 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2413}
2414
2415static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2416{
2417 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2418
2419 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2420
2421
2422 ucd_req_ptr->header.dword_0 =
2423 UPIU_HEADER_DWORD(
2424 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2425
2426 ucd_req_ptr->header.dword_1 = 0;
2427 ucd_req_ptr->header.dword_2 = 0;
2428
2429 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2430}
2431
2432
2433
2434
2435
2436
2437
2438static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2439 struct ufshcd_lrb *lrbp)
2440{
2441 u8 upiu_flags;
2442 int ret = 0;
2443
2444 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2445 (hba->ufs_version == UFSHCI_VERSION_11))
2446 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2447 else
2448 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2449
2450 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2451 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2452 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2453 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2454 ufshcd_prepare_utp_nop_upiu(lrbp);
2455 else
2456 ret = -EINVAL;
2457
2458 return ret;
2459}
2460
2461
2462
2463
2464
2465
2466
2467static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2468{
2469 u8 upiu_flags;
2470 int ret = 0;
2471
2472 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2473 (hba->ufs_version == UFSHCI_VERSION_11))
2474 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2475 else
2476 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2477
2478 if (likely(lrbp->cmd)) {
2479 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2480 lrbp->cmd->sc_data_direction);
2481 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2482 } else {
2483 ret = -EINVAL;
2484 }
2485
2486 return ret;
2487}
2488
2489
2490
2491
2492
2493
2494
2495static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2496{
2497 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2498}
2499
2500static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2501{
2502 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2503 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2504 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2505 i * sizeof(struct utp_transfer_cmd_desc);
2506 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2507 response_upiu);
2508 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2509
2510 lrb->utr_descriptor_ptr = utrdlp + i;
2511 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2512 i * sizeof(struct utp_transfer_req_desc);
2513 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2514 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2515 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2516 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2517 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2518 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2519}
2520
2521
2522
2523
2524
2525
2526
2527
2528static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2529{
2530 struct ufshcd_lrb *lrbp;
2531 struct ufs_hba *hba;
2532 unsigned long flags;
2533 int tag;
2534 int err = 0;
2535
2536 hba = shost_priv(host);
2537
2538 tag = cmd->request->tag;
2539 if (!ufshcd_valid_tag(hba, tag)) {
2540 dev_err(hba->dev,
2541 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2542 __func__, tag, cmd, cmd->request);
2543 BUG();
2544 }
2545
2546 if (!down_read_trylock(&hba->clk_scaling_lock))
2547 return SCSI_MLQUEUE_HOST_BUSY;
2548
2549 hba->req_abort_count = 0;
2550
2551 err = ufshcd_hold(hba, true);
2552 if (err) {
2553 err = SCSI_MLQUEUE_HOST_BUSY;
2554 goto out;
2555 }
2556 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2557 (hba->clk_gating.state != CLKS_ON));
2558
2559 lrbp = &hba->lrb[tag];
2560
2561 WARN_ON(lrbp->cmd);
2562 lrbp->cmd = cmd;
2563 lrbp->sense_bufflen = UFS_SENSE_SIZE;
2564 lrbp->sense_buffer = cmd->sense_buffer;
2565 lrbp->task_tag = tag;
2566 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2567 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2568
2569 ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2570
2571 lrbp->req_abort_skip = false;
2572
2573 ufshcd_comp_scsi_upiu(hba, lrbp);
2574
2575 err = ufshcd_map_sg(hba, lrbp);
2576 if (err) {
2577 lrbp->cmd = NULL;
2578 ufshcd_release(hba);
2579 goto out;
2580 }
2581
2582 wmb();
2583
2584 spin_lock_irqsave(hba->host->host_lock, flags);
2585 switch (hba->ufshcd_state) {
2586 case UFSHCD_STATE_OPERATIONAL:
2587 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2588 break;
2589 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600 if (hba->pm_op_in_progress) {
2601 hba->force_reset = true;
2602 set_host_byte(cmd, DID_BAD_TARGET);
2603 goto out_compl_cmd;
2604 }
2605 fallthrough;
2606 case UFSHCD_STATE_RESET:
2607 err = SCSI_MLQUEUE_HOST_BUSY;
2608 goto out_compl_cmd;
2609 case UFSHCD_STATE_ERROR:
2610 set_host_byte(cmd, DID_ERROR);
2611 goto out_compl_cmd;
2612 default:
2613 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2614 __func__, hba->ufshcd_state);
2615 set_host_byte(cmd, DID_BAD_TARGET);
2616 goto out_compl_cmd;
2617 }
2618 ufshcd_send_command(hba, tag);
2619 spin_unlock_irqrestore(hba->host->host_lock, flags);
2620 goto out;
2621
2622out_compl_cmd:
2623 scsi_dma_unmap(lrbp->cmd);
2624 lrbp->cmd = NULL;
2625 spin_unlock_irqrestore(hba->host->host_lock, flags);
2626 ufshcd_release(hba);
2627 if (!err)
2628 cmd->scsi_done(cmd);
2629out:
2630 up_read(&hba->clk_scaling_lock);
2631 return err;
2632}
2633
2634static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2635 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2636{
2637 lrbp->cmd = NULL;
2638 lrbp->sense_bufflen = 0;
2639 lrbp->sense_buffer = NULL;
2640 lrbp->task_tag = tag;
2641 lrbp->lun = 0;
2642 lrbp->intr_cmd = true;
2643 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2644 hba->dev_cmd.type = cmd_type;
2645
2646 return ufshcd_compose_devman_upiu(hba, lrbp);
2647}
2648
2649static int
2650ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2651{
2652 int err = 0;
2653 unsigned long flags;
2654 u32 mask = 1 << tag;
2655
2656
2657 spin_lock_irqsave(hba->host->host_lock, flags);
2658 ufshcd_utrl_clear(hba, tag);
2659 spin_unlock_irqrestore(hba->host->host_lock, flags);
2660
2661
2662
2663
2664
2665 err = ufshcd_wait_for_register(hba,
2666 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2667 mask, ~mask, 1000, 1000);
2668
2669 return err;
2670}
2671
2672static int
2673ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2674{
2675 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2676
2677
2678 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2679 UPIU_RSP_CODE_OFFSET;
2680 return query_res->response;
2681}
2682
2683
2684
2685
2686
2687
2688static int
2689ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2690{
2691 int resp;
2692 int err = 0;
2693
2694 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2695 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2696
2697 switch (resp) {
2698 case UPIU_TRANSACTION_NOP_IN:
2699 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2700 err = -EINVAL;
2701 dev_err(hba->dev, "%s: unexpected response %x\n",
2702 __func__, resp);
2703 }
2704 break;
2705 case UPIU_TRANSACTION_QUERY_RSP:
2706 err = ufshcd_check_query_response(hba, lrbp);
2707 if (!err)
2708 err = ufshcd_copy_query_response(hba, lrbp);
2709 break;
2710 case UPIU_TRANSACTION_REJECT_UPIU:
2711
2712 err = -EPERM;
2713 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2714 __func__);
2715 break;
2716 default:
2717 err = -EINVAL;
2718 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2719 __func__, resp);
2720 break;
2721 }
2722
2723 return err;
2724}
2725
2726static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2727 struct ufshcd_lrb *lrbp, int max_timeout)
2728{
2729 int err = 0;
2730 unsigned long time_left;
2731 unsigned long flags;
2732
2733 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2734 msecs_to_jiffies(max_timeout));
2735
2736
2737 wmb();
2738 spin_lock_irqsave(hba->host->host_lock, flags);
2739 hba->dev_cmd.complete = NULL;
2740 if (likely(time_left)) {
2741 err = ufshcd_get_tr_ocs(lrbp);
2742 if (!err)
2743 err = ufshcd_dev_cmd_completion(hba, lrbp);
2744 }
2745 spin_unlock_irqrestore(hba->host->host_lock, flags);
2746
2747 if (!time_left) {
2748 err = -ETIMEDOUT;
2749 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2750 __func__, lrbp->task_tag);
2751 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2752
2753 err = -EAGAIN;
2754
2755
2756
2757
2758
2759 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2760 }
2761
2762 return err;
2763}
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2775 enum dev_cmd_type cmd_type, int timeout)
2776{
2777 struct request_queue *q = hba->cmd_queue;
2778 struct request *req;
2779 struct ufshcd_lrb *lrbp;
2780 int err;
2781 int tag;
2782 struct completion wait;
2783 unsigned long flags;
2784
2785 down_read(&hba->clk_scaling_lock);
2786
2787
2788
2789
2790
2791
2792 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2793 if (IS_ERR(req)) {
2794 err = PTR_ERR(req);
2795 goto out_unlock;
2796 }
2797 tag = req->tag;
2798 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
2799
2800 init_completion(&wait);
2801 lrbp = &hba->lrb[tag];
2802 WARN_ON(lrbp->cmd);
2803 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2804 if (unlikely(err))
2805 goto out_put_tag;
2806
2807 hba->dev_cmd.complete = &wait;
2808
2809 ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2810
2811 wmb();
2812 spin_lock_irqsave(hba->host->host_lock, flags);
2813 ufshcd_send_command(hba, tag);
2814 spin_unlock_irqrestore(hba->host->host_lock, flags);
2815
2816 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2817
2818 ufshcd_add_query_upiu_trace(hba, tag,
2819 err ? "query_complete_err" : "query_complete");
2820
2821out_put_tag:
2822 blk_put_request(req);
2823out_unlock:
2824 up_read(&hba->clk_scaling_lock);
2825 return err;
2826}
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838static inline void ufshcd_init_query(struct ufs_hba *hba,
2839 struct ufs_query_req **request, struct ufs_query_res **response,
2840 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2841{
2842 *request = &hba->dev_cmd.query.request;
2843 *response = &hba->dev_cmd.query.response;
2844 memset(*request, 0, sizeof(struct ufs_query_req));
2845 memset(*response, 0, sizeof(struct ufs_query_res));
2846 (*request)->upiu_req.opcode = opcode;
2847 (*request)->upiu_req.idn = idn;
2848 (*request)->upiu_req.index = index;
2849 (*request)->upiu_req.selector = selector;
2850}
2851
2852static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2853 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
2854{
2855 int ret;
2856 int retries;
2857
2858 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2859 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
2860 if (ret)
2861 dev_dbg(hba->dev,
2862 "%s: failed with error %d, retries %d\n",
2863 __func__, ret, retries);
2864 else
2865 break;
2866 }
2867
2868 if (ret)
2869 dev_err(hba->dev,
2870 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2871 __func__, opcode, idn, ret, retries);
2872 return ret;
2873}
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2886 enum flag_idn idn, u8 index, bool *flag_res)
2887{
2888 struct ufs_query_req *request = NULL;
2889 struct ufs_query_res *response = NULL;
2890 int err, selector = 0;
2891 int timeout = QUERY_REQ_TIMEOUT;
2892
2893 BUG_ON(!hba);
2894
2895 ufshcd_hold(hba, false);
2896 mutex_lock(&hba->dev_cmd.lock);
2897 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2898 selector);
2899
2900 switch (opcode) {
2901 case UPIU_QUERY_OPCODE_SET_FLAG:
2902 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2903 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2904 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2905 break;
2906 case UPIU_QUERY_OPCODE_READ_FLAG:
2907 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2908 if (!flag_res) {
2909
2910 dev_err(hba->dev, "%s: Invalid argument for read request\n",
2911 __func__);
2912 err = -EINVAL;
2913 goto out_unlock;
2914 }
2915 break;
2916 default:
2917 dev_err(hba->dev,
2918 "%s: Expected query flag opcode but got = %d\n",
2919 __func__, opcode);
2920 err = -EINVAL;
2921 goto out_unlock;
2922 }
2923
2924 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2925
2926 if (err) {
2927 dev_err(hba->dev,
2928 "%s: Sending flag query for idn %d failed, err = %d\n",
2929 __func__, idn, err);
2930 goto out_unlock;
2931 }
2932
2933 if (flag_res)
2934 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2935 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2936
2937out_unlock:
2938 mutex_unlock(&hba->dev_cmd.lock);
2939 ufshcd_release(hba);
2940 return err;
2941}
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2955 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2956{
2957 struct ufs_query_req *request = NULL;
2958 struct ufs_query_res *response = NULL;
2959 int err;
2960
2961 BUG_ON(!hba);
2962
2963 ufshcd_hold(hba, false);
2964 if (!attr_val) {
2965 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2966 __func__, opcode);
2967 err = -EINVAL;
2968 goto out;
2969 }
2970
2971 mutex_lock(&hba->dev_cmd.lock);
2972 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2973 selector);
2974
2975 switch (opcode) {
2976 case UPIU_QUERY_OPCODE_WRITE_ATTR:
2977 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2978 request->upiu_req.value = cpu_to_be32(*attr_val);
2979 break;
2980 case UPIU_QUERY_OPCODE_READ_ATTR:
2981 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2982 break;
2983 default:
2984 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2985 __func__, opcode);
2986 err = -EINVAL;
2987 goto out_unlock;
2988 }
2989
2990 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2991
2992 if (err) {
2993 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2994 __func__, opcode, idn, index, err);
2995 goto out_unlock;
2996 }
2997
2998 *attr_val = be32_to_cpu(response->upiu_res.value);
2999
3000out_unlock:
3001 mutex_unlock(&hba->dev_cmd.lock);
3002out:
3003 ufshcd_release(hba);
3004 return err;
3005}
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020static int ufshcd_query_attr_retry(struct ufs_hba *hba,
3021 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3022 u32 *attr_val)
3023{
3024 int ret = 0;
3025 u32 retries;
3026
3027 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3028 ret = ufshcd_query_attr(hba, opcode, idn, index,
3029 selector, attr_val);
3030 if (ret)
3031 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3032 __func__, ret, retries);
3033 else
3034 break;
3035 }
3036
3037 if (ret)
3038 dev_err(hba->dev,
3039 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3040 __func__, idn, ret, QUERY_REQ_RETRIES);
3041 return ret;
3042}
3043
3044static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3045 enum query_opcode opcode, enum desc_idn idn, u8 index,
3046 u8 selector, u8 *desc_buf, int *buf_len)
3047{
3048 struct ufs_query_req *request = NULL;
3049 struct ufs_query_res *response = NULL;
3050 int err;
3051
3052 BUG_ON(!hba);
3053
3054 ufshcd_hold(hba, false);
3055 if (!desc_buf) {
3056 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3057 __func__, opcode);
3058 err = -EINVAL;
3059 goto out;
3060 }
3061
3062 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3063 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3064 __func__, *buf_len);
3065 err = -EINVAL;
3066 goto out;
3067 }
3068
3069 mutex_lock(&hba->dev_cmd.lock);
3070 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3071 selector);
3072 hba->dev_cmd.query.descriptor = desc_buf;
3073 request->upiu_req.length = cpu_to_be16(*buf_len);
3074
3075 switch (opcode) {
3076 case UPIU_QUERY_OPCODE_WRITE_DESC:
3077 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3078 break;
3079 case UPIU_QUERY_OPCODE_READ_DESC:
3080 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3081 break;
3082 default:
3083 dev_err(hba->dev,
3084 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3085 __func__, opcode);
3086 err = -EINVAL;
3087 goto out_unlock;
3088 }
3089
3090 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3091
3092 if (err) {
3093 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3094 __func__, opcode, idn, index, err);
3095 goto out_unlock;
3096 }
3097
3098 *buf_len = be16_to_cpu(response->upiu_res.length);
3099
3100out_unlock:
3101 hba->dev_cmd.query.descriptor = NULL;
3102 mutex_unlock(&hba->dev_cmd.lock);
3103out:
3104 ufshcd_release(hba);
3105 return err;
3106}
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3123 enum query_opcode opcode,
3124 enum desc_idn idn, u8 index,
3125 u8 selector,
3126 u8 *desc_buf, int *buf_len)
3127{
3128 int err;
3129 int retries;
3130
3131 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3132 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3133 selector, desc_buf, buf_len);
3134 if (!err || err == -EINVAL)
3135 break;
3136 }
3137
3138 return err;
3139}
3140
3141
3142
3143
3144
3145
3146
3147void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3148 int *desc_len)
3149{
3150 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3151 desc_id == QUERY_DESC_IDN_RFU_1)
3152 *desc_len = 0;
3153 else
3154 *desc_len = hba->desc_size[desc_id];
3155}
3156EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3157
3158static void ufshcd_update_desc_length(struct ufs_hba *hba,
3159 enum desc_idn desc_id, int desc_index,
3160 unsigned char desc_len)
3161{
3162 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3163 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3164
3165
3166
3167
3168
3169 hba->desc_size[desc_id] = desc_len;
3170}
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183int ufshcd_read_desc_param(struct ufs_hba *hba,
3184 enum desc_idn desc_id,
3185 int desc_index,
3186 u8 param_offset,
3187 u8 *param_read_buf,
3188 u8 param_size)
3189{
3190 int ret;
3191 u8 *desc_buf;
3192 int buff_len;
3193 bool is_kmalloc = true;
3194
3195
3196 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3197 return -EINVAL;
3198
3199
3200 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3201 if (!buff_len) {
3202 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3203 return -EINVAL;
3204 }
3205
3206 if (param_offset >= buff_len) {
3207 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3208 __func__, param_offset, desc_id, buff_len);
3209 return -EINVAL;
3210 }
3211
3212
3213 if (param_offset != 0 || param_size < buff_len) {
3214 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3215 if (!desc_buf)
3216 return -ENOMEM;
3217 } else {
3218 desc_buf = param_read_buf;
3219 is_kmalloc = false;
3220 }
3221
3222
3223 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3224 desc_id, desc_index, 0,
3225 desc_buf, &buff_len);
3226
3227 if (ret) {
3228 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3229 __func__, desc_id, desc_index, param_offset, ret);
3230 goto out;
3231 }
3232
3233
3234 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3235 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3236 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3237 ret = -EINVAL;
3238 goto out;
3239 }
3240
3241
3242 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3243 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3244
3245 if (is_kmalloc) {
3246
3247 if (param_offset + param_size > buff_len)
3248 param_size = buff_len - param_offset;
3249 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3250 }
3251out:
3252 if (is_kmalloc)
3253 kfree(desc_buf);
3254 return ret;
3255}
3256
3257
3258
3259
3260
3261
3262
3263
3264struct uc_string_id {
3265 u8 len;
3266 u8 type;
3267 wchar_t uc[];
3268} __packed;
3269
3270
3271static inline char ufshcd_remove_non_printable(u8 ch)
3272{
3273 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3274}
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3291 u8 **buf, bool ascii)
3292{
3293 struct uc_string_id *uc_str;
3294 u8 *str;
3295 int ret;
3296
3297 if (!buf)
3298 return -EINVAL;
3299
3300 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3301 if (!uc_str)
3302 return -ENOMEM;
3303
3304 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3305 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3306 if (ret < 0) {
3307 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3308 QUERY_REQ_RETRIES, ret);
3309 str = NULL;
3310 goto out;
3311 }
3312
3313 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3314 dev_dbg(hba->dev, "String Desc is of zero length\n");
3315 str = NULL;
3316 ret = 0;
3317 goto out;
3318 }
3319
3320 if (ascii) {
3321 ssize_t ascii_len;
3322 int i;
3323
3324 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3325 str = kzalloc(ascii_len, GFP_KERNEL);
3326 if (!str) {
3327 ret = -ENOMEM;
3328 goto out;
3329 }
3330
3331
3332
3333
3334
3335 ret = utf16s_to_utf8s(uc_str->uc,
3336 uc_str->len - QUERY_DESC_HDR_SIZE,
3337 UTF16_BIG_ENDIAN, str, ascii_len);
3338
3339
3340 for (i = 0; i < ret; i++)
3341 str[i] = ufshcd_remove_non_printable(str[i]);
3342
3343 str[ret++] = '\0';
3344
3345 } else {
3346 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3347 if (!str) {
3348 ret = -ENOMEM;
3349 goto out;
3350 }
3351 ret = uc_str->len;
3352 }
3353out:
3354 *buf = str;
3355 kfree(uc_str);
3356 return ret;
3357}
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3370 int lun,
3371 enum unit_desc_param param_offset,
3372 u8 *param_read_buf,
3373 u32 param_size)
3374{
3375
3376
3377
3378
3379 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3380 return -EOPNOTSUPP;
3381
3382 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3383 param_offset, param_read_buf, param_size);
3384}
3385
3386static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3387{
3388 int err = 0;
3389 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3390
3391 if (hba->dev_info.wspecversion >= 0x300) {
3392 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3393 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3394 &gating_wait);
3395 if (err)
3396 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3397 err, gating_wait);
3398
3399 if (gating_wait == 0) {
3400 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3401 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3402 gating_wait);
3403 }
3404
3405 hba->dev_info.clk_gating_wait_us = gating_wait;
3406 }
3407
3408 return err;
3409}
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424static int ufshcd_memory_alloc(struct ufs_hba *hba)
3425{
3426 size_t utmrdl_size, utrdl_size, ucdl_size;
3427
3428
3429 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3430 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3431 ucdl_size,
3432 &hba->ucdl_dma_addr,
3433 GFP_KERNEL);
3434
3435
3436
3437
3438
3439
3440
3441 if (!hba->ucdl_base_addr ||
3442 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3443 dev_err(hba->dev,
3444 "Command Descriptor Memory allocation failed\n");
3445 goto out;
3446 }
3447
3448
3449
3450
3451
3452 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3453 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3454 utrdl_size,
3455 &hba->utrdl_dma_addr,
3456 GFP_KERNEL);
3457 if (!hba->utrdl_base_addr ||
3458 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3459 dev_err(hba->dev,
3460 "Transfer Descriptor Memory allocation failed\n");
3461 goto out;
3462 }
3463
3464
3465
3466
3467
3468 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3469 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3470 utmrdl_size,
3471 &hba->utmrdl_dma_addr,
3472 GFP_KERNEL);
3473 if (!hba->utmrdl_base_addr ||
3474 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3475 dev_err(hba->dev,
3476 "Task Management Descriptor Memory allocation failed\n");
3477 goto out;
3478 }
3479
3480
3481 hba->lrb = devm_kcalloc(hba->dev,
3482 hba->nutrs, sizeof(struct ufshcd_lrb),
3483 GFP_KERNEL);
3484 if (!hba->lrb) {
3485 dev_err(hba->dev, "LRB Memory allocation failed\n");
3486 goto out;
3487 }
3488 return 0;
3489out:
3490 return -ENOMEM;
3491}
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3507{
3508 struct utp_transfer_req_desc *utrdlp;
3509 dma_addr_t cmd_desc_dma_addr;
3510 dma_addr_t cmd_desc_element_addr;
3511 u16 response_offset;
3512 u16 prdt_offset;
3513 int cmd_desc_size;
3514 int i;
3515
3516 utrdlp = hba->utrdl_base_addr;
3517
3518 response_offset =
3519 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3520 prdt_offset =
3521 offsetof(struct utp_transfer_cmd_desc, prd_table);
3522
3523 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3524 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3525
3526 for (i = 0; i < hba->nutrs; i++) {
3527
3528 cmd_desc_element_addr =
3529 (cmd_desc_dma_addr + (cmd_desc_size * i));
3530 utrdlp[i].command_desc_base_addr_lo =
3531 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3532 utrdlp[i].command_desc_base_addr_hi =
3533 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3534
3535
3536 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3537 utrdlp[i].response_upiu_offset =
3538 cpu_to_le16(response_offset);
3539 utrdlp[i].prd_table_offset =
3540 cpu_to_le16(prdt_offset);
3541 utrdlp[i].response_upiu_length =
3542 cpu_to_le16(ALIGNED_UPIU_SIZE);
3543 } else {
3544 utrdlp[i].response_upiu_offset =
3545 cpu_to_le16(response_offset >> 2);
3546 utrdlp[i].prd_table_offset =
3547 cpu_to_le16(prdt_offset >> 2);
3548 utrdlp[i].response_upiu_length =
3549 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3550 }
3551
3552 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3553 }
3554}
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3568{
3569 struct uic_command uic_cmd = {0};
3570 int ret;
3571
3572 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3573
3574 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3575 if (ret)
3576 dev_dbg(hba->dev,
3577 "dme-link-startup: error code %d\n", ret);
3578 return ret;
3579}
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589static int ufshcd_dme_reset(struct ufs_hba *hba)
3590{
3591 struct uic_command uic_cmd = {0};
3592 int ret;
3593
3594 uic_cmd.command = UIC_CMD_DME_RESET;
3595
3596 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3597 if (ret)
3598 dev_err(hba->dev,
3599 "dme-reset: error code %d\n", ret);
3600
3601 return ret;
3602}
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612static int ufshcd_dme_enable(struct ufs_hba *hba)
3613{
3614 struct uic_command uic_cmd = {0};
3615 int ret;
3616
3617 uic_cmd.command = UIC_CMD_DME_ENABLE;
3618
3619 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3620 if (ret)
3621 dev_err(hba->dev,
3622 "dme-reset: error code %d\n", ret);
3623
3624 return ret;
3625}
3626
3627static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3628{
3629 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3630 unsigned long min_sleep_time_us;
3631
3632 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3633 return;
3634
3635
3636
3637
3638
3639 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3640 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3641 } else {
3642 unsigned long delta =
3643 (unsigned long) ktime_to_us(
3644 ktime_sub(ktime_get(),
3645 hba->last_dme_cmd_tstamp));
3646
3647 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3648 min_sleep_time_us =
3649 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3650 else
3651 return;
3652 }
3653
3654
3655 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3656}
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3669 u8 attr_set, u32 mib_val, u8 peer)
3670{
3671 struct uic_command uic_cmd = {0};
3672 static const char *const action[] = {
3673 "dme-set",
3674 "dme-peer-set"
3675 };
3676 const char *set = action[!!peer];
3677 int ret;
3678 int retries = UFS_UIC_COMMAND_RETRIES;
3679
3680 uic_cmd.command = peer ?
3681 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3682 uic_cmd.argument1 = attr_sel;
3683 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3684 uic_cmd.argument3 = mib_val;
3685
3686 do {
3687
3688 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3689 if (ret)
3690 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3691 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3692 } while (ret && peer && --retries);
3693
3694 if (ret)
3695 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3696 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3697 UFS_UIC_COMMAND_RETRIES - retries);
3698
3699 return ret;
3700}
3701EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3713 u32 *mib_val, u8 peer)
3714{
3715 struct uic_command uic_cmd = {0};
3716 static const char *const action[] = {
3717 "dme-get",
3718 "dme-peer-get"
3719 };
3720 const char *get = action[!!peer];
3721 int ret;
3722 int retries = UFS_UIC_COMMAND_RETRIES;
3723 struct ufs_pa_layer_attr orig_pwr_info;
3724 struct ufs_pa_layer_attr temp_pwr_info;
3725 bool pwr_mode_change = false;
3726
3727 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3728 orig_pwr_info = hba->pwr_info;
3729 temp_pwr_info = orig_pwr_info;
3730
3731 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3732 orig_pwr_info.pwr_rx == FAST_MODE) {
3733 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3734 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3735 pwr_mode_change = true;
3736 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3737 orig_pwr_info.pwr_rx == SLOW_MODE) {
3738 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3739 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3740 pwr_mode_change = true;
3741 }
3742 if (pwr_mode_change) {
3743 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3744 if (ret)
3745 goto out;
3746 }
3747 }
3748
3749 uic_cmd.command = peer ?
3750 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3751 uic_cmd.argument1 = attr_sel;
3752
3753 do {
3754
3755 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3756 if (ret)
3757 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3758 get, UIC_GET_ATTR_ID(attr_sel), ret);
3759 } while (ret && peer && --retries);
3760
3761 if (ret)
3762 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3763 get, UIC_GET_ATTR_ID(attr_sel),
3764 UFS_UIC_COMMAND_RETRIES - retries);
3765
3766 if (mib_val && !ret)
3767 *mib_val = uic_cmd.argument3;
3768
3769 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3770 && pwr_mode_change)
3771 ufshcd_change_power_mode(hba, &orig_pwr_info);
3772out:
3773 return ret;
3774}
3775EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3794{
3795 struct completion uic_async_done;
3796 unsigned long flags;
3797 u8 status;
3798 int ret;
3799 bool reenable_intr = false;
3800
3801 mutex_lock(&hba->uic_cmd_mutex);
3802 init_completion(&uic_async_done);
3803 ufshcd_add_delay_before_dme_cmd(hba);
3804
3805 spin_lock_irqsave(hba->host->host_lock, flags);
3806 if (ufshcd_is_link_broken(hba)) {
3807 ret = -ENOLINK;
3808 goto out_unlock;
3809 }
3810 hba->uic_async_done = &uic_async_done;
3811 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3812 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3813
3814
3815
3816
3817 wmb();
3818 reenable_intr = true;
3819 }
3820 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3821 spin_unlock_irqrestore(hba->host->host_lock, flags);
3822 if (ret) {
3823 dev_err(hba->dev,
3824 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3825 cmd->command, cmd->argument3, ret);
3826 goto out;
3827 }
3828
3829 if (!wait_for_completion_timeout(hba->uic_async_done,
3830 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3831 dev_err(hba->dev,
3832 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3833 cmd->command, cmd->argument3);
3834
3835 if (!cmd->cmd_active) {
3836 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
3837 __func__);
3838 goto check_upmcrs;
3839 }
3840
3841 ret = -ETIMEDOUT;
3842 goto out;
3843 }
3844
3845check_upmcrs:
3846 status = ufshcd_get_upmcrs(hba);
3847 if (status != PWR_LOCAL) {
3848 dev_err(hba->dev,
3849 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
3850 cmd->command, status);
3851 ret = (status != PWR_OK) ? status : -1;
3852 }
3853out:
3854 if (ret) {
3855 ufshcd_print_host_state(hba);
3856 ufshcd_print_pwr_info(hba);
3857 ufshcd_print_host_regs(hba);
3858 }
3859
3860 spin_lock_irqsave(hba->host->host_lock, flags);
3861 hba->active_uic_cmd = NULL;
3862 hba->uic_async_done = NULL;
3863 if (reenable_intr)
3864 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
3865 if (ret) {
3866 ufshcd_set_link_broken(hba);
3867 ufshcd_schedule_eh_work(hba);
3868 }
3869out_unlock:
3870 spin_unlock_irqrestore(hba->host->host_lock, flags);
3871 mutex_unlock(&hba->uic_cmd_mutex);
3872
3873 return ret;
3874}
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3885{
3886 struct uic_command uic_cmd = {0};
3887 int ret;
3888
3889 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3890 ret = ufshcd_dme_set(hba,
3891 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3892 if (ret) {
3893 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3894 __func__, ret);
3895 goto out;
3896 }
3897 }
3898
3899 uic_cmd.command = UIC_CMD_DME_SET;
3900 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3901 uic_cmd.argument3 = mode;
3902 ufshcd_hold(hba, false);
3903 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3904 ufshcd_release(hba);
3905
3906out:
3907 return ret;
3908}
3909
3910int ufshcd_link_recovery(struct ufs_hba *hba)
3911{
3912 int ret;
3913 unsigned long flags;
3914
3915 spin_lock_irqsave(hba->host->host_lock, flags);
3916 hba->ufshcd_state = UFSHCD_STATE_RESET;
3917 ufshcd_set_eh_in_progress(hba);
3918 spin_unlock_irqrestore(hba->host->host_lock, flags);
3919
3920
3921 ufshcd_vops_device_reset(hba);
3922
3923 ret = ufshcd_host_reset_and_restore(hba);
3924
3925 spin_lock_irqsave(hba->host->host_lock, flags);
3926 if (ret)
3927 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3928 ufshcd_clear_eh_in_progress(hba);
3929 spin_unlock_irqrestore(hba->host->host_lock, flags);
3930
3931 if (ret)
3932 dev_err(hba->dev, "%s: link recovery failed, err %d",
3933 __func__, ret);
3934
3935 return ret;
3936}
3937EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
3938
3939static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3940{
3941 int ret;
3942 struct uic_command uic_cmd = {0};
3943 ktime_t start = ktime_get();
3944
3945 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3946
3947 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
3948 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3949 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3950 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3951
3952 if (ret)
3953 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3954 __func__, ret);
3955 else
3956 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3957 POST_CHANGE);
3958
3959 return ret;
3960}
3961
3962int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3963{
3964 struct uic_command uic_cmd = {0};
3965 int ret;
3966 ktime_t start = ktime_get();
3967
3968 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3969
3970 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3971 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3972 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3973 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3974
3975 if (ret) {
3976 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3977 __func__, ret);
3978 } else {
3979 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3980 POST_CHANGE);
3981 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3982 hba->ufs_stats.hibern8_exit_cnt++;
3983 }
3984
3985 return ret;
3986}
3987EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
3988
3989void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
3990{
3991 unsigned long flags;
3992 bool update = false;
3993
3994 if (!ufshcd_is_auto_hibern8_supported(hba))
3995 return;
3996
3997 spin_lock_irqsave(hba->host->host_lock, flags);
3998 if (hba->ahit != ahit) {
3999 hba->ahit = ahit;
4000 update = true;
4001 }
4002 spin_unlock_irqrestore(hba->host->host_lock, flags);
4003
4004 if (update && !pm_runtime_suspended(hba->dev)) {
4005 pm_runtime_get_sync(hba->dev);
4006 ufshcd_hold(hba, false);
4007 ufshcd_auto_hibern8_enable(hba);
4008 ufshcd_release(hba);
4009 pm_runtime_put(hba->dev);
4010 }
4011}
4012EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4013
4014void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4015{
4016 unsigned long flags;
4017
4018 if (!ufshcd_is_auto_hibern8_supported(hba))
4019 return;
4020
4021 spin_lock_irqsave(hba->host->host_lock, flags);
4022 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4023 spin_unlock_irqrestore(hba->host->host_lock, flags);
4024}
4025
4026
4027
4028
4029
4030
4031static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4032{
4033 hba->pwr_info.gear_rx = UFS_PWM_G1;
4034 hba->pwr_info.gear_tx = UFS_PWM_G1;
4035 hba->pwr_info.lane_rx = 1;
4036 hba->pwr_info.lane_tx = 1;
4037 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4038 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4039 hba->pwr_info.hs_rate = 0;
4040}
4041
4042
4043
4044
4045
4046static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4047{
4048 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4049
4050 if (hba->max_pwr_info.is_valid)
4051 return 0;
4052
4053 pwr_info->pwr_tx = FAST_MODE;
4054 pwr_info->pwr_rx = FAST_MODE;
4055 pwr_info->hs_rate = PA_HS_MODE_B;
4056
4057
4058 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4059 &pwr_info->lane_rx);
4060 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4061 &pwr_info->lane_tx);
4062
4063 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4064 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4065 __func__,
4066 pwr_info->lane_rx,
4067 pwr_info->lane_tx);
4068 return -EINVAL;
4069 }
4070
4071
4072
4073
4074
4075
4076 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4077 if (!pwr_info->gear_rx) {
4078 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4079 &pwr_info->gear_rx);
4080 if (!pwr_info->gear_rx) {
4081 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4082 __func__, pwr_info->gear_rx);
4083 return -EINVAL;
4084 }
4085 pwr_info->pwr_rx = SLOW_MODE;
4086 }
4087
4088 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4089 &pwr_info->gear_tx);
4090 if (!pwr_info->gear_tx) {
4091 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4092 &pwr_info->gear_tx);
4093 if (!pwr_info->gear_tx) {
4094 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4095 __func__, pwr_info->gear_tx);
4096 return -EINVAL;
4097 }
4098 pwr_info->pwr_tx = SLOW_MODE;
4099 }
4100
4101 hba->max_pwr_info.is_valid = true;
4102 return 0;
4103}
4104
4105static int ufshcd_change_power_mode(struct ufs_hba *hba,
4106 struct ufs_pa_layer_attr *pwr_mode)
4107{
4108 int ret;
4109
4110
4111 if (!hba->force_pmc &&
4112 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4113 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4114 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4115 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4116 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4117 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4118 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4119 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4120 return 0;
4121 }
4122
4123
4124
4125
4126
4127
4128
4129 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4130 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4131 pwr_mode->lane_rx);
4132 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4133 pwr_mode->pwr_rx == FAST_MODE)
4134 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4135 else
4136 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4137
4138 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4139 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4140 pwr_mode->lane_tx);
4141 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4142 pwr_mode->pwr_tx == FAST_MODE)
4143 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4144 else
4145 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4146
4147 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4148 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4149 pwr_mode->pwr_rx == FAST_MODE ||
4150 pwr_mode->pwr_tx == FAST_MODE)
4151 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4152 pwr_mode->hs_rate);
4153
4154 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4155 DL_FC0ProtectionTimeOutVal_Default);
4156 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4157 DL_TC0ReplayTimeOutVal_Default);
4158 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4159 DL_AFC0ReqTimeOutVal_Default);
4160 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4161 DL_FC1ProtectionTimeOutVal_Default);
4162 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4163 DL_TC1ReplayTimeOutVal_Default);
4164 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4165 DL_AFC1ReqTimeOutVal_Default);
4166
4167 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4168 DL_FC0ProtectionTimeOutVal_Default);
4169 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4170 DL_TC0ReplayTimeOutVal_Default);
4171 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4172 DL_AFC0ReqTimeOutVal_Default);
4173
4174 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4175 | pwr_mode->pwr_tx);
4176
4177 if (ret) {
4178 dev_err(hba->dev,
4179 "%s: power mode change failed %d\n", __func__, ret);
4180 } else {
4181 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4182 pwr_mode);
4183
4184 memcpy(&hba->pwr_info, pwr_mode,
4185 sizeof(struct ufs_pa_layer_attr));
4186 }
4187
4188 return ret;
4189}
4190
4191
4192
4193
4194
4195
4196int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4197 struct ufs_pa_layer_attr *desired_pwr_mode)
4198{
4199 struct ufs_pa_layer_attr final_params = { 0 };
4200 int ret;
4201
4202 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4203 desired_pwr_mode, &final_params);
4204
4205 if (ret)
4206 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4207
4208 ret = ufshcd_change_power_mode(hba, &final_params);
4209
4210 return ret;
4211}
4212EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4213
4214
4215
4216
4217
4218
4219
4220static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4221{
4222 int err;
4223 bool flag_res = true;
4224 ktime_t timeout;
4225
4226 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4227 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4228 if (err) {
4229 dev_err(hba->dev,
4230 "%s setting fDeviceInit flag failed with error %d\n",
4231 __func__, err);
4232 goto out;
4233 }
4234
4235
4236 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4237 do {
4238 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4239 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4240 if (!flag_res)
4241 break;
4242 usleep_range(5000, 10000);
4243 } while (ktime_before(ktime_get(), timeout));
4244
4245 if (err) {
4246 dev_err(hba->dev,
4247 "%s reading fDeviceInit flag failed with error %d\n",
4248 __func__, err);
4249 } else if (flag_res) {
4250 dev_err(hba->dev,
4251 "%s fDeviceInit was not cleared by the device\n",
4252 __func__);
4253 err = -EBUSY;
4254 }
4255out:
4256 return err;
4257}
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271int ufshcd_make_hba_operational(struct ufs_hba *hba)
4272{
4273 int err = 0;
4274 u32 reg;
4275
4276
4277 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4278
4279
4280 if (ufshcd_is_intr_aggr_allowed(hba))
4281 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4282 else
4283 ufshcd_disable_intr_aggr(hba);
4284
4285
4286 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4287 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4288 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4289 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4290 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4291 REG_UTP_TASK_REQ_LIST_BASE_L);
4292 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4293 REG_UTP_TASK_REQ_LIST_BASE_H);
4294
4295
4296
4297
4298
4299 wmb();
4300
4301
4302
4303
4304 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4305 if (!(ufshcd_get_lists_status(reg))) {
4306 ufshcd_enable_run_stop_reg(hba);
4307 } else {
4308 dev_err(hba->dev,
4309 "Host controller not ready to process requests");
4310 err = -EIO;
4311 }
4312
4313 return err;
4314}
4315EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4316
4317
4318
4319
4320
4321static inline void ufshcd_hba_stop(struct ufs_hba *hba)
4322{
4323 unsigned long flags;
4324 int err;
4325
4326
4327
4328
4329
4330 spin_lock_irqsave(hba->host->host_lock, flags);
4331 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4332 spin_unlock_irqrestore(hba->host->host_lock, flags);
4333
4334 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4335 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4336 10, 1);
4337 if (err)
4338 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4339}
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4352{
4353 int retry;
4354
4355 if (!ufshcd_is_hba_active(hba))
4356
4357 ufshcd_hba_stop(hba);
4358
4359
4360 ufshcd_set_link_off(hba);
4361
4362 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4363
4364
4365 ufshcd_hba_start(hba);
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4378
4379
4380 retry = 50;
4381 while (ufshcd_is_hba_active(hba)) {
4382 if (retry) {
4383 retry--;
4384 } else {
4385 dev_err(hba->dev,
4386 "Controller enable failed\n");
4387 return -EIO;
4388 }
4389 usleep_range(1000, 1100);
4390 }
4391
4392
4393 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4394
4395 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4396
4397 return 0;
4398}
4399
4400int ufshcd_hba_enable(struct ufs_hba *hba)
4401{
4402 int ret;
4403
4404 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4405 ufshcd_set_link_off(hba);
4406 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4407
4408
4409 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4410 ret = ufshcd_dme_reset(hba);
4411 if (!ret) {
4412 ret = ufshcd_dme_enable(hba);
4413 if (!ret)
4414 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4415 if (ret)
4416 dev_err(hba->dev,
4417 "Host controller enable failed with non-hce\n");
4418 }
4419 } else {
4420 ret = ufshcd_hba_execute_hce(hba);
4421 }
4422
4423 return ret;
4424}
4425EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4426
4427static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4428{
4429 int tx_lanes = 0, i, err = 0;
4430
4431 if (!peer)
4432 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4433 &tx_lanes);
4434 else
4435 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4436 &tx_lanes);
4437 for (i = 0; i < tx_lanes; i++) {
4438 if (!peer)
4439 err = ufshcd_dme_set(hba,
4440 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4441 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4442 0);
4443 else
4444 err = ufshcd_dme_peer_set(hba,
4445 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4446 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4447 0);
4448 if (err) {
4449 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4450 __func__, peer, i, err);
4451 break;
4452 }
4453 }
4454
4455 return err;
4456}
4457
4458static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4459{
4460 return ufshcd_disable_tx_lcc(hba, true);
4461}
4462
4463void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
4464 u32 reg)
4465{
4466 reg_hist->reg[reg_hist->pos] = reg;
4467 reg_hist->tstamp[reg_hist->pos] = ktime_get();
4468 reg_hist->pos = (reg_hist->pos + 1) % UFS_ERR_REG_HIST_LENGTH;
4469}
4470EXPORT_SYMBOL_GPL(ufshcd_update_reg_hist);
4471
4472
4473
4474
4475
4476
4477
4478static int ufshcd_link_startup(struct ufs_hba *hba)
4479{
4480 int ret;
4481 int retries = DME_LINKSTARTUP_RETRIES;
4482 bool link_startup_again = false;
4483
4484
4485
4486
4487
4488 if (!ufshcd_is_ufs_dev_active(hba))
4489 link_startup_again = true;
4490
4491link_startup:
4492 do {
4493 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4494
4495 ret = ufshcd_dme_link_startup(hba);
4496
4497
4498 if (!ret && !ufshcd_is_device_present(hba)) {
4499 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4500 0);
4501 dev_err(hba->dev, "%s: Device not present\n", __func__);
4502 ret = -ENXIO;
4503 goto out;
4504 }
4505
4506
4507
4508
4509
4510
4511 if (ret && ufshcd_hba_enable(hba)) {
4512 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4513 (u32)ret);
4514 goto out;
4515 }
4516 } while (ret && retries--);
4517
4518 if (ret) {
4519
4520 ufshcd_update_reg_hist(&hba->ufs_stats.link_startup_err,
4521 (u32)ret);
4522 goto out;
4523 }
4524
4525 if (link_startup_again) {
4526 link_startup_again = false;
4527 retries = DME_LINKSTARTUP_RETRIES;
4528 goto link_startup;
4529 }
4530
4531
4532 ufshcd_init_pwr_info(hba);
4533 ufshcd_print_pwr_info(hba);
4534
4535 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4536 ret = ufshcd_disable_device_tx_lcc(hba);
4537 if (ret)
4538 goto out;
4539 }
4540
4541
4542 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4543 if (ret)
4544 goto out;
4545
4546
4547 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4548 ret = ufshcd_make_hba_operational(hba);
4549out:
4550 if (ret) {
4551 dev_err(hba->dev, "link startup failed %d\n", ret);
4552 ufshcd_print_host_state(hba);
4553 ufshcd_print_pwr_info(hba);
4554 ufshcd_print_host_regs(hba);
4555 }
4556 return ret;
4557}
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4570{
4571 int err = 0;
4572 int retries;
4573
4574 ufshcd_hold(hba, false);
4575 mutex_lock(&hba->dev_cmd.lock);
4576 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4577 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4578 NOP_OUT_TIMEOUT);
4579
4580 if (!err || err == -ETIMEDOUT)
4581 break;
4582
4583 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4584 }
4585 mutex_unlock(&hba->dev_cmd.lock);
4586 ufshcd_release(hba);
4587
4588 if (err)
4589 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4590 return err;
4591}
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4603{
4604 int ret = 0;
4605 u8 lun_qdepth;
4606 struct ufs_hba *hba;
4607
4608 hba = shost_priv(sdev->host);
4609
4610 lun_qdepth = hba->nutrs;
4611 ret = ufshcd_read_unit_desc_param(hba,
4612 ufshcd_scsi_to_upiu_lun(sdev->lun),
4613 UNIT_DESC_PARAM_LU_Q_DEPTH,
4614 &lun_qdepth,
4615 sizeof(lun_qdepth));
4616
4617
4618 if (ret == -EOPNOTSUPP)
4619 lun_qdepth = 1;
4620 else if (!lun_qdepth)
4621
4622 lun_qdepth = hba->nutrs;
4623 else
4624 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4625
4626 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4627 __func__, lun_qdepth);
4628 scsi_change_queue_depth(sdev, lun_qdepth);
4629}
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4643 u8 lun,
4644 u8 *b_lu_write_protect)
4645{
4646 int ret;
4647
4648 if (!b_lu_write_protect)
4649 ret = -EINVAL;
4650
4651
4652
4653
4654
4655 else if (lun >= hba->dev_info.max_lu_supported)
4656 ret = -ENOTSUPP;
4657 else
4658 ret = ufshcd_read_unit_desc_param(hba,
4659 lun,
4660 UNIT_DESC_PARAM_LU_WR_PROTECT,
4661 b_lu_write_protect,
4662 sizeof(*b_lu_write_protect));
4663 return ret;
4664}
4665
4666
4667
4668
4669
4670
4671
4672
4673static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4674 struct scsi_device *sdev)
4675{
4676 if (hba->dev_info.f_power_on_wp_en &&
4677 !hba->dev_info.is_lu_power_on_wp) {
4678 u8 b_lu_write_protect;
4679
4680 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4681 &b_lu_write_protect) &&
4682 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4683 hba->dev_info.is_lu_power_on_wp = true;
4684 }
4685}
4686
4687
4688
4689
4690
4691
4692
4693static int ufshcd_slave_alloc(struct scsi_device *sdev)
4694{
4695 struct ufs_hba *hba;
4696
4697 hba = shost_priv(sdev->host);
4698
4699
4700 sdev->use_10_for_ms = 1;
4701
4702
4703 sdev->set_dbd_for_ms = 1;
4704
4705
4706 sdev->allow_restart = 1;
4707
4708
4709 sdev->no_report_opcodes = 1;
4710
4711
4712 sdev->no_write_same = 1;
4713
4714 ufshcd_set_queue_depth(sdev);
4715
4716 ufshcd_get_lu_power_on_wp_status(hba, sdev);
4717
4718 return 0;
4719}
4720
4721
4722
4723
4724
4725
4726
4727
4728static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4729{
4730 struct ufs_hba *hba = shost_priv(sdev->host);
4731
4732 if (depth > hba->nutrs)
4733 depth = hba->nutrs;
4734 return scsi_change_queue_depth(sdev, depth);
4735}
4736
4737
4738
4739
4740
4741static int ufshcd_slave_configure(struct scsi_device *sdev)
4742{
4743 struct ufs_hba *hba = shost_priv(sdev->host);
4744 struct request_queue *q = sdev->request_queue;
4745
4746 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4747
4748 if (ufshcd_is_rpm_autosuspend_allowed(hba))
4749 sdev->rpm_autosuspend = 1;
4750
4751 ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
4752
4753 return 0;
4754}
4755
4756
4757
4758
4759
4760static void ufshcd_slave_destroy(struct scsi_device *sdev)
4761{
4762 struct ufs_hba *hba;
4763
4764 hba = shost_priv(sdev->host);
4765
4766 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4767 unsigned long flags;
4768
4769 spin_lock_irqsave(hba->host->host_lock, flags);
4770 hba->sdev_ufs_device = NULL;
4771 spin_unlock_irqrestore(hba->host->host_lock, flags);
4772 }
4773}
4774
4775
4776
4777
4778
4779
4780
4781
4782static inline int
4783ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4784{
4785 int result = 0;
4786
4787 switch (scsi_status) {
4788 case SAM_STAT_CHECK_CONDITION:
4789 ufshcd_copy_sense_data(lrbp);
4790 fallthrough;
4791 case SAM_STAT_GOOD:
4792 result |= DID_OK << 16 |
4793 COMMAND_COMPLETE << 8 |
4794 scsi_status;
4795 break;
4796 case SAM_STAT_TASK_SET_FULL:
4797 case SAM_STAT_BUSY:
4798 case SAM_STAT_TASK_ABORTED:
4799 ufshcd_copy_sense_data(lrbp);
4800 result |= scsi_status;
4801 break;
4802 default:
4803 result |= DID_ERROR << 16;
4804 break;
4805 }
4806
4807 return result;
4808}
4809
4810
4811
4812
4813
4814
4815
4816
4817static inline int
4818ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4819{
4820 int result = 0;
4821 int scsi_status;
4822 int ocs;
4823
4824
4825 ocs = ufshcd_get_tr_ocs(lrbp);
4826
4827 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
4828 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
4829 MASK_RSP_UPIU_RESULT)
4830 ocs = OCS_SUCCESS;
4831 }
4832
4833 switch (ocs) {
4834 case OCS_SUCCESS:
4835 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
4836 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
4837 switch (result) {
4838 case UPIU_TRANSACTION_RESPONSE:
4839
4840
4841
4842
4843 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4844
4845
4846
4847
4848
4849 scsi_status = result & MASK_SCSI_STATUS;
4850 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864 if (!hba->pm_op_in_progress &&
4865 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr) &&
4866 schedule_work(&hba->eeh_work)) {
4867
4868
4869
4870
4871
4872 pm_runtime_get_noresume(hba->dev);
4873 }
4874 break;
4875 case UPIU_TRANSACTION_REJECT_UPIU:
4876
4877 result = DID_ERROR << 16;
4878 dev_err(hba->dev,
4879 "Reject UPIU not fully implemented\n");
4880 break;
4881 default:
4882 dev_err(hba->dev,
4883 "Unexpected request response code = %x\n",
4884 result);
4885 result = DID_ERROR << 16;
4886 break;
4887 }
4888 break;
4889 case OCS_ABORTED:
4890 result |= DID_ABORT << 16;
4891 break;
4892 case OCS_INVALID_COMMAND_STATUS:
4893 result |= DID_REQUEUE << 16;
4894 break;
4895 case OCS_INVALID_CMD_TABLE_ATTR:
4896 case OCS_INVALID_PRDT_ATTR:
4897 case OCS_MISMATCH_DATA_BUF_SIZE:
4898 case OCS_MISMATCH_RESP_UPIU_SIZE:
4899 case OCS_PEER_COMM_FAILURE:
4900 case OCS_FATAL_ERROR:
4901 case OCS_DEVICE_FATAL_ERROR:
4902 case OCS_INVALID_CRYPTO_CONFIG:
4903 case OCS_GENERAL_CRYPTO_ERROR:
4904 default:
4905 result |= DID_ERROR << 16;
4906 dev_err(hba->dev,
4907 "OCS error from controller = %x for tag %d\n",
4908 ocs, lrbp->task_tag);
4909 ufshcd_print_host_regs(hba);
4910 ufshcd_print_host_state(hba);
4911 break;
4912 }
4913
4914 if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
4915 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
4916 return result;
4917}
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
4929{
4930 irqreturn_t retval = IRQ_NONE;
4931
4932 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
4933 hba->active_uic_cmd->argument2 |=
4934 ufshcd_get_uic_cmd_result(hba);
4935 hba->active_uic_cmd->argument3 =
4936 ufshcd_get_dme_attr_val(hba);
4937 if (!hba->uic_async_done)
4938 hba->active_uic_cmd->cmd_active = 0;
4939 complete(&hba->active_uic_cmd->done);
4940 retval = IRQ_HANDLED;
4941 }
4942
4943 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
4944 hba->active_uic_cmd->cmd_active = 0;
4945 complete(hba->uic_async_done);
4946 retval = IRQ_HANDLED;
4947 }
4948
4949 if (retval == IRQ_HANDLED)
4950 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
4951 "complete");
4952 return retval;
4953}
4954
4955
4956
4957
4958
4959
4960static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4961 unsigned long completed_reqs)
4962{
4963 struct ufshcd_lrb *lrbp;
4964 struct scsi_cmnd *cmd;
4965 int result;
4966 int index;
4967
4968 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4969 lrbp = &hba->lrb[index];
4970 lrbp->compl_time_stamp = ktime_get();
4971 cmd = lrbp->cmd;
4972 if (cmd) {
4973 ufshcd_add_command_trace(hba, index, "complete");
4974 result = ufshcd_transfer_rsp_status(hba, lrbp);
4975 scsi_dma_unmap(cmd);
4976 cmd->result = result;
4977
4978 lrbp->cmd = NULL;
4979
4980 cmd->scsi_done(cmd);
4981 __ufshcd_release(hba);
4982 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4983 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
4984 if (hba->dev_cmd.complete) {
4985 ufshcd_add_command_trace(hba, index,
4986 "dev_complete");
4987 complete(hba->dev_cmd.complete);
4988 }
4989 }
4990 if (ufshcd_is_clkscaling_supported(hba))
4991 hba->clk_scaling.active_reqs--;
4992 }
4993
4994
4995 hba->outstanding_reqs ^= completed_reqs;
4996
4997 ufshcd_clk_scaling_update_busy(hba);
4998}
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5009{
5010 unsigned long completed_reqs;
5011 u32 tr_doorbell;
5012
5013
5014
5015
5016
5017
5018
5019
5020 if (ufshcd_is_intr_aggr_allowed(hba) &&
5021 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5022 ufshcd_reset_intr_aggr(hba);
5023
5024 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5025 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
5026
5027 if (completed_reqs) {
5028 __ufshcd_transfer_req_compl(hba, completed_reqs);
5029 return IRQ_HANDLED;
5030 } else {
5031 return IRQ_NONE;
5032 }
5033}
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5046{
5047 int err = 0;
5048 u32 val;
5049
5050 if (!(hba->ee_ctrl_mask & mask))
5051 goto out;
5052
5053 val = hba->ee_ctrl_mask & ~mask;
5054 val &= MASK_EE_STATUS;
5055 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5056 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5057 if (!err)
5058 hba->ee_ctrl_mask &= ~mask;
5059out:
5060 return err;
5061}
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5074{
5075 int err = 0;
5076 u32 val;
5077
5078 if (hba->ee_ctrl_mask & mask)
5079 goto out;
5080
5081 val = hba->ee_ctrl_mask | mask;
5082 val &= MASK_EE_STATUS;
5083 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5084 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
5085 if (!err)
5086 hba->ee_ctrl_mask |= mask;
5087out:
5088 return err;
5089}
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5103{
5104 int err = 0;
5105
5106 if (hba->auto_bkops_enabled)
5107 goto out;
5108
5109 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5110 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5111 if (err) {
5112 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5113 __func__, err);
5114 goto out;
5115 }
5116
5117 hba->auto_bkops_enabled = true;
5118 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5119
5120
5121 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5122 if (err)
5123 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5124 __func__, err);
5125out:
5126 return err;
5127}
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5142{
5143 int err = 0;
5144
5145 if (!hba->auto_bkops_enabled)
5146 goto out;
5147
5148
5149
5150
5151
5152 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5153 if (err) {
5154 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5155 __func__, err);
5156 goto out;
5157 }
5158
5159 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5160 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5161 if (err) {
5162 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5163 __func__, err);
5164 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5165 goto out;
5166 }
5167
5168 hba->auto_bkops_enabled = false;
5169 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5170 hba->is_urgent_bkops_lvl_checked = false;
5171out:
5172 return err;
5173}
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5185{
5186 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5187 hba->auto_bkops_enabled = false;
5188 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5189 ufshcd_enable_auto_bkops(hba);
5190 } else {
5191 hba->auto_bkops_enabled = true;
5192 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5193 ufshcd_disable_auto_bkops(hba);
5194 }
5195 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5196 hba->is_urgent_bkops_lvl_checked = false;
5197}
5198
5199static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5200{
5201 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5202 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5203}
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5222 enum bkops_status status)
5223{
5224 int err;
5225 u32 curr_status = 0;
5226
5227 err = ufshcd_get_bkops_status(hba, &curr_status);
5228 if (err) {
5229 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5230 __func__, err);
5231 goto out;
5232 } else if (curr_status > BKOPS_STATUS_MAX) {
5233 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5234 __func__, curr_status);
5235 err = -EINVAL;
5236 goto out;
5237 }
5238
5239 if (curr_status >= status)
5240 err = ufshcd_enable_auto_bkops(hba);
5241 else
5242 err = ufshcd_disable_auto_bkops(hba);
5243out:
5244 return err;
5245}
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5258{
5259 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5260}
5261
5262static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5263{
5264 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5265 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5266}
5267
5268static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5269{
5270 int err;
5271 u32 curr_status = 0;
5272
5273 if (hba->is_urgent_bkops_lvl_checked)
5274 goto enable_auto_bkops;
5275
5276 err = ufshcd_get_bkops_status(hba, &curr_status);
5277 if (err) {
5278 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5279 __func__, err);
5280 goto out;
5281 }
5282
5283
5284
5285
5286
5287
5288
5289 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5290 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5291 __func__, curr_status);
5292
5293 hba->urgent_bkops_lvl = curr_status;
5294 hba->is_urgent_bkops_lvl_checked = true;
5295 }
5296
5297enable_auto_bkops:
5298 err = ufshcd_enable_auto_bkops(hba);
5299out:
5300 if (err < 0)
5301 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5302 __func__, err);
5303}
5304
5305static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
5306{
5307 int ret;
5308 u8 index;
5309 enum query_opcode opcode;
5310
5311 if (!ufshcd_is_wb_allowed(hba))
5312 return 0;
5313
5314 if (!(enable ^ hba->wb_enabled))
5315 return 0;
5316 if (enable)
5317 opcode = UPIU_QUERY_OPCODE_SET_FLAG;
5318 else
5319 opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5320
5321 index = ufshcd_wb_get_query_index(hba);
5322 ret = ufshcd_query_flag_retry(hba, opcode,
5323 QUERY_FLAG_IDN_WB_EN, index, NULL);
5324 if (ret) {
5325 dev_err(hba->dev, "%s write booster %s failed %d\n",
5326 __func__, enable ? "enable" : "disable", ret);
5327 return ret;
5328 }
5329
5330 hba->wb_enabled = enable;
5331 dev_dbg(hba->dev, "%s write booster %s %d\n",
5332 __func__, enable ? "enable" : "disable", ret);
5333
5334 return ret;
5335}
5336
5337static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5338{
5339 int val;
5340 u8 index;
5341
5342 if (set)
5343 val = UPIU_QUERY_OPCODE_SET_FLAG;
5344 else
5345 val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
5346
5347 index = ufshcd_wb_get_query_index(hba);
5348 return ufshcd_query_flag_retry(hba, val,
5349 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
5350 index, NULL);
5351}
5352
5353static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5354{
5355 if (hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)
5356 return;
5357
5358 if (enable)
5359 ufshcd_wb_buf_flush_enable(hba);
5360 else
5361 ufshcd_wb_buf_flush_disable(hba);
5362
5363}
5364
5365static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba)
5366{
5367 int ret;
5368 u8 index;
5369
5370 if (!ufshcd_is_wb_allowed(hba) || hba->wb_buf_flush_enabled)
5371 return 0;
5372
5373 index = ufshcd_wb_get_query_index(hba);
5374 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5375 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5376 index, NULL);
5377 if (ret)
5378 dev_err(hba->dev, "%s WB - buf flush enable failed %d\n",
5379 __func__, ret);
5380 else
5381 hba->wb_buf_flush_enabled = true;
5382
5383 dev_dbg(hba->dev, "WB - Flush enabled: %d\n", ret);
5384 return ret;
5385}
5386
5387static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba)
5388{
5389 int ret;
5390 u8 index;
5391
5392 if (!ufshcd_is_wb_allowed(hba) || !hba->wb_buf_flush_enabled)
5393 return 0;
5394
5395 index = ufshcd_wb_get_query_index(hba);
5396 ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5397 QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
5398 index, NULL);
5399 if (ret) {
5400 dev_warn(hba->dev, "%s: WB - buf flush disable failed %d\n",
5401 __func__, ret);
5402 } else {
5403 hba->wb_buf_flush_enabled = false;
5404 dev_dbg(hba->dev, "WB - Flush disabled: %d\n", ret);
5405 }
5406
5407 return ret;
5408}
5409
5410static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5411 u32 avail_buf)
5412{
5413 u32 cur_buf;
5414 int ret;
5415 u8 index;
5416
5417 index = ufshcd_wb_get_query_index(hba);
5418 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5419 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
5420 index, 0, &cur_buf);
5421 if (ret) {
5422 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5423 __func__, ret);
5424 return false;
5425 }
5426
5427 if (!cur_buf) {
5428 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5429 cur_buf);
5430 return false;
5431 }
5432
5433 if (avail_buf < hba->vps->wb_flush_threshold)
5434 return true;
5435
5436 return false;
5437}
5438
5439static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
5440{
5441 int ret;
5442 u32 avail_buf;
5443 u8 index;
5444
5445 if (!ufshcd_is_wb_allowed(hba))
5446 return false;
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458 index = ufshcd_wb_get_query_index(hba);
5459 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5460 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
5461 index, 0, &avail_buf);
5462 if (ret) {
5463 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5464 __func__, ret);
5465 return false;
5466 }
5467
5468 if (!hba->dev_info.b_presrv_uspc_en) {
5469 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
5470 return true;
5471 return false;
5472 }
5473
5474 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5475}
5476
5477static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5478{
5479 struct ufs_hba *hba = container_of(to_delayed_work(work),
5480 struct ufs_hba,
5481 rpm_dev_flush_recheck_work);
5482
5483
5484
5485
5486
5487
5488 pm_runtime_get_sync(hba->dev);
5489 pm_runtime_put_sync(hba->dev);
5490}
5491
5492
5493
5494
5495
5496
5497
5498
5499static void ufshcd_exception_event_handler(struct work_struct *work)
5500{
5501 struct ufs_hba *hba;
5502 int err;
5503 u32 status = 0;
5504 hba = container_of(work, struct ufs_hba, eeh_work);
5505
5506 pm_runtime_get_sync(hba->dev);
5507 ufshcd_scsi_block_requests(hba);
5508 err = ufshcd_get_ee_status(hba, &status);
5509 if (err) {
5510 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5511 __func__, err);
5512 goto out;
5513 }
5514
5515 status &= hba->ee_ctrl_mask;
5516
5517 if (status & MASK_EE_URGENT_BKOPS)
5518 ufshcd_bkops_exception_event_handler(hba);
5519
5520out:
5521 ufshcd_scsi_unblock_requests(hba);
5522
5523
5524
5525
5526
5527
5528 pm_runtime_put_noidle(hba->dev);
5529 pm_runtime_put(hba->dev);
5530 return;
5531}
5532
5533
5534static void ufshcd_complete_requests(struct ufs_hba *hba)
5535{
5536 ufshcd_transfer_req_compl(hba);
5537 ufshcd_tmc_handler(hba);
5538}
5539
5540
5541
5542
5543
5544
5545
5546
5547static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5548{
5549 unsigned long flags;
5550 bool err_handling = true;
5551
5552 spin_lock_irqsave(hba->host->host_lock, flags);
5553
5554
5555
5556
5557 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5558 goto out;
5559
5560 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5561 ((hba->saved_err & UIC_ERROR) &&
5562 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5563 goto out;
5564
5565 if ((hba->saved_err & UIC_ERROR) &&
5566 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5567 int err;
5568
5569
5570
5571 spin_unlock_irqrestore(hba->host->host_lock, flags);
5572 msleep(50);
5573 spin_lock_irqsave(hba->host->host_lock, flags);
5574
5575
5576
5577
5578
5579 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5580 ((hba->saved_err & UIC_ERROR) &&
5581 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5582 goto out;
5583
5584
5585
5586
5587
5588
5589
5590
5591 spin_unlock_irqrestore(hba->host->host_lock, flags);
5592 err = ufshcd_verify_dev_init(hba);
5593 spin_lock_irqsave(hba->host->host_lock, flags);
5594
5595 if (err)
5596 goto out;
5597
5598
5599 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5600 hba->saved_err &= ~UIC_ERROR;
5601
5602 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5603 if (!hba->saved_uic_err)
5604 err_handling = false;
5605 }
5606out:
5607 spin_unlock_irqrestore(hba->host->host_lock, flags);
5608 return err_handling;
5609}
5610
5611
5612static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5613{
5614 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5615 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5616}
5617
5618
5619static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5620{
5621
5622 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5623 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5624 ufshcd_is_saved_err_fatal(hba))
5625 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5626 else
5627 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5628 queue_work(hba->eh_wq, &hba->eh_work);
5629 }
5630}
5631
5632static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5633{
5634 pm_runtime_get_sync(hba->dev);
5635 if (pm_runtime_suspended(hba->dev)) {
5636
5637
5638
5639
5640
5641 ufshcd_setup_hba_vreg(hba, true);
5642 ufshcd_enable_irq(hba);
5643 ufshcd_setup_vreg(hba, true);
5644 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5645 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5646 ufshcd_hold(hba, false);
5647 if (!ufshcd_is_clkgating_allowed(hba))
5648 ufshcd_setup_clocks(hba, true);
5649 ufshcd_release(hba);
5650 ufshcd_vops_resume(hba, UFS_RUNTIME_PM);
5651 } else {
5652 ufshcd_hold(hba, false);
5653 if (hba->clk_scaling.is_allowed) {
5654 cancel_work_sync(&hba->clk_scaling.suspend_work);
5655 cancel_work_sync(&hba->clk_scaling.resume_work);
5656 ufshcd_suspend_clkscaling(hba);
5657 }
5658 }
5659}
5660
5661static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5662{
5663 ufshcd_release(hba);
5664 if (hba->clk_scaling.is_allowed)
5665 ufshcd_resume_clkscaling(hba);
5666 pm_runtime_put(hba->dev);
5667}
5668
5669static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5670{
5671 return (hba->ufshcd_state == UFSHCD_STATE_ERROR ||
5672 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
5673 ufshcd_is_link_broken(hba))));
5674}
5675
5676#ifdef CONFIG_PM
5677static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5678{
5679 struct Scsi_Host *shost = hba->host;
5680 struct scsi_device *sdev;
5681 struct request_queue *q;
5682 int ret;
5683
5684
5685
5686
5687
5688 ret = pm_runtime_set_active(hba->dev);
5689
5690
5691
5692
5693
5694
5695 if (!ret) {
5696 shost_for_each_device(sdev, shost) {
5697 q = sdev->request_queue;
5698 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
5699 q->rpm_status == RPM_SUSPENDING))
5700 pm_request_resume(q->dev);
5701 }
5702 }
5703}
5704#else
5705static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
5706{
5707}
5708#endif
5709
5710static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
5711{
5712 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
5713 u32 mode;
5714
5715 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
5716
5717 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
5718 return true;
5719
5720 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
5721 return true;
5722
5723 return false;
5724}
5725
5726
5727
5728
5729
5730static void ufshcd_err_handler(struct work_struct *work)
5731{
5732 struct ufs_hba *hba;
5733 unsigned long flags;
5734 bool err_xfer = false;
5735 bool err_tm = false;
5736 int err = 0, pmc_err;
5737 int tag;
5738 bool needs_reset = false, needs_restore = false;
5739
5740 hba = container_of(work, struct ufs_hba, eh_work);
5741
5742 spin_lock_irqsave(hba->host->host_lock, flags);
5743 if (ufshcd_err_handling_should_stop(hba)) {
5744 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5745 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5746 spin_unlock_irqrestore(hba->host->host_lock, flags);
5747 return;
5748 }
5749 ufshcd_set_eh_in_progress(hba);
5750 spin_unlock_irqrestore(hba->host->host_lock, flags);
5751 ufshcd_err_handling_prepare(hba);
5752 spin_lock_irqsave(hba->host->host_lock, flags);
5753 ufshcd_scsi_block_requests(hba);
5754
5755
5756
5757
5758 if (ufshcd_err_handling_should_stop(hba)) {
5759 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
5760 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5761 goto out;
5762 }
5763 hba->ufshcd_state = UFSHCD_STATE_RESET;
5764
5765
5766 ufshcd_complete_requests(hba);
5767
5768 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5769 bool ret;
5770
5771 spin_unlock_irqrestore(hba->host->host_lock, flags);
5772
5773 ret = ufshcd_quirk_dl_nac_errors(hba);
5774 spin_lock_irqsave(hba->host->host_lock, flags);
5775 if (!ret && !hba->force_reset && ufshcd_is_link_active(hba))
5776 goto skip_err_handling;
5777 }
5778
5779 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5780 ufshcd_is_saved_err_fatal(hba) ||
5781 ((hba->saved_err & UIC_ERROR) &&
5782 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5783 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5784 needs_reset = true;
5785
5786 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
5787 (hba->saved_uic_err &&
5788 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
5789 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
5790
5791 spin_unlock_irqrestore(hba->host->host_lock, flags);
5792 ufshcd_print_host_state(hba);
5793 ufshcd_print_pwr_info(hba);
5794 ufshcd_print_host_regs(hba);
5795 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5796 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
5797 spin_lock_irqsave(hba->host->host_lock, flags);
5798 }
5799
5800
5801
5802
5803
5804
5805 if (needs_reset)
5806 goto do_reset;
5807
5808
5809
5810
5811
5812 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
5813 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5814 if (!hba->saved_uic_err)
5815 hba->saved_err &= ~UIC_ERROR;
5816 spin_unlock_irqrestore(hba->host->host_lock, flags);
5817 if (ufshcd_is_pwr_mode_restore_needed(hba))
5818 needs_restore = true;
5819 spin_lock_irqsave(hba->host->host_lock, flags);
5820 if (!hba->saved_err && !needs_restore)
5821 goto skip_err_handling;
5822 }
5823
5824 hba->silence_err_logs = true;
5825
5826 spin_unlock_irqrestore(hba->host->host_lock, flags);
5827
5828 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5829 if (ufshcd_try_to_abort_task(hba, tag)) {
5830 err_xfer = true;
5831 goto lock_skip_pending_xfer_clear;
5832 }
5833 }
5834
5835
5836 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5837 if (ufshcd_clear_tm_cmd(hba, tag)) {
5838 err_tm = true;
5839 goto lock_skip_pending_xfer_clear;
5840 }
5841 }
5842
5843lock_skip_pending_xfer_clear:
5844 spin_lock_irqsave(hba->host->host_lock, flags);
5845
5846
5847 ufshcd_complete_requests(hba);
5848 hba->silence_err_logs = false;
5849
5850 if (err_xfer || err_tm) {
5851 needs_reset = true;
5852 goto do_reset;
5853 }
5854
5855
5856
5857
5858
5859 if (needs_restore) {
5860 spin_unlock_irqrestore(hba->host->host_lock, flags);
5861
5862
5863
5864
5865 down_write(&hba->clk_scaling_lock);
5866 hba->force_pmc = true;
5867 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
5868 if (pmc_err) {
5869 needs_reset = true;
5870 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
5871 __func__, pmc_err);
5872 }
5873 hba->force_pmc = false;
5874 ufshcd_print_pwr_info(hba);
5875 up_write(&hba->clk_scaling_lock);
5876 spin_lock_irqsave(hba->host->host_lock, flags);
5877 }
5878
5879do_reset:
5880
5881 if (needs_reset) {
5882 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5883
5884
5885
5886
5887
5888
5889
5890
5891 if (hba->outstanding_reqs == max_doorbells)
5892 __ufshcd_transfer_req_compl(hba,
5893 (1UL << (hba->nutrs - 1)));
5894
5895 hba->force_reset = false;
5896 spin_unlock_irqrestore(hba->host->host_lock, flags);
5897 err = ufshcd_reset_and_restore(hba);
5898 if (err)
5899 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
5900 __func__, err);
5901 else
5902 ufshcd_recover_pm_error(hba);
5903 spin_lock_irqsave(hba->host->host_lock, flags);
5904 }
5905
5906skip_err_handling:
5907 if (!needs_reset) {
5908 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
5909 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5910 if (hba->saved_err || hba->saved_uic_err)
5911 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5912 __func__, hba->saved_err, hba->saved_uic_err);
5913 }
5914
5915out:
5916 ufshcd_clear_eh_in_progress(hba);
5917 spin_unlock_irqrestore(hba->host->host_lock, flags);
5918 ufshcd_scsi_unblock_requests(hba);
5919 ufshcd_err_handling_unprepare(hba);
5920}
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
5931{
5932 u32 reg;
5933 irqreturn_t retval = IRQ_NONE;
5934
5935
5936 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5937 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
5938 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
5939 ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
5940
5941
5942
5943
5944 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
5945 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
5946 __func__);
5947
5948
5949 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
5950 struct uic_command *cmd = NULL;
5951
5952 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
5953 if (hba->uic_async_done && hba->active_uic_cmd)
5954 cmd = hba->active_uic_cmd;
5955
5956
5957
5958
5959 if (cmd && (cmd->command == UIC_CMD_DME_SET))
5960 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
5961 }
5962 retval |= IRQ_HANDLED;
5963 }
5964
5965
5966 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
5967 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
5968 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
5969 ufshcd_update_reg_hist(&hba->ufs_stats.dl_err, reg);
5970
5971 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5972 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5973 else if (hba->dev_quirks &
5974 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5975 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5976 hba->uic_error |=
5977 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5978 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5979 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5980 }
5981 retval |= IRQ_HANDLED;
5982 }
5983
5984
5985 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
5986 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
5987 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
5988 ufshcd_update_reg_hist(&hba->ufs_stats.nl_err, reg);
5989 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
5990 retval |= IRQ_HANDLED;
5991 }
5992
5993 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
5994 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
5995 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
5996 ufshcd_update_reg_hist(&hba->ufs_stats.tl_err, reg);
5997 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
5998 retval |= IRQ_HANDLED;
5999 }
6000
6001 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6002 if ((reg & UIC_DME_ERROR) &&
6003 (reg & UIC_DME_ERROR_CODE_MASK)) {
6004 ufshcd_update_reg_hist(&hba->ufs_stats.dme_err, reg);
6005 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6006 retval |= IRQ_HANDLED;
6007 }
6008
6009 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6010 __func__, hba->uic_error);
6011 return retval;
6012}
6013
6014static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
6015 u32 intr_mask)
6016{
6017 if (!ufshcd_is_auto_hibern8_supported(hba) ||
6018 !ufshcd_is_auto_hibern8_enabled(hba))
6019 return false;
6020
6021 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
6022 return false;
6023
6024 if (hba->active_uic_cmd &&
6025 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
6026 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
6027 return false;
6028
6029 return true;
6030}
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
6041{
6042 bool queue_eh_work = false;
6043 irqreturn_t retval = IRQ_NONE;
6044
6045 if (hba->errors & INT_FATAL_ERRORS) {
6046 ufshcd_update_reg_hist(&hba->ufs_stats.fatal_err, hba->errors);
6047 queue_eh_work = true;
6048 }
6049
6050 if (hba->errors & UIC_ERROR) {
6051 hba->uic_error = 0;
6052 retval = ufshcd_update_uic_error(hba);
6053 if (hba->uic_error)
6054 queue_eh_work = true;
6055 }
6056
6057 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6058 dev_err(hba->dev,
6059 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6060 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6061 "Enter" : "Exit",
6062 hba->errors, ufshcd_get_upmcrs(hba));
6063 ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
6064 hba->errors);
6065 ufshcd_set_link_broken(hba);
6066 queue_eh_work = true;
6067 }
6068
6069 if (queue_eh_work) {
6070
6071
6072
6073
6074 hba->saved_err |= hba->errors;
6075 hba->saved_uic_err |= hba->uic_error;
6076
6077
6078 if ((hba->saved_err & (INT_FATAL_ERRORS)) ||
6079 (hba->saved_uic_err &&
6080 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6081 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6082 __func__, hba->saved_err,
6083 hba->saved_uic_err);
6084 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6085 "host_regs: ");
6086 ufshcd_print_pwr_info(hba);
6087 }
6088 ufshcd_schedule_eh_work(hba);
6089 retval |= IRQ_HANDLED;
6090 }
6091
6092
6093
6094
6095
6096
6097 return retval;
6098}
6099
6100struct ctm_info {
6101 struct ufs_hba *hba;
6102 unsigned long pending;
6103 unsigned int ncpl;
6104};
6105
6106static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
6107{
6108 struct ctm_info *const ci = priv;
6109 struct completion *c;
6110
6111 WARN_ON_ONCE(reserved);
6112 if (test_bit(req->tag, &ci->pending))
6113 return true;
6114 ci->ncpl++;
6115 c = req->end_io_data;
6116 if (c)
6117 complete(c);
6118 return true;
6119}
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6130{
6131 struct request_queue *q = hba->tmf_queue;
6132 struct ctm_info ci = {
6133 .hba = hba,
6134 .pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL),
6135 };
6136
6137 blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
6138 return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
6139}
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6151{
6152 irqreturn_t retval = IRQ_NONE;
6153
6154 hba->errors = UFSHCD_ERROR_MASK & intr_status;
6155
6156 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
6157 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
6158
6159 if (hba->errors)
6160 retval |= ufshcd_check_errors(hba);
6161
6162 if (intr_status & UFSHCD_UIC_MASK)
6163 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6164
6165 if (intr_status & UTP_TASK_REQ_COMPL)
6166 retval |= ufshcd_tmc_handler(hba);
6167
6168 if (intr_status & UTP_TRANSFER_REQ_COMPL)
6169 retval |= ufshcd_transfer_req_compl(hba);
6170
6171 return retval;
6172}
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183static irqreturn_t ufshcd_intr(int irq, void *__hba)
6184{
6185 u32 intr_status, enabled_intr_status = 0;
6186 irqreturn_t retval = IRQ_NONE;
6187 struct ufs_hba *hba = __hba;
6188 int retries = hba->nutrs;
6189
6190 spin_lock(hba->host->host_lock);
6191 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6192 hba->ufs_stats.last_intr_status = intr_status;
6193 hba->ufs_stats.last_intr_ts = ktime_get();
6194
6195
6196
6197
6198
6199
6200
6201 while (intr_status && retries--) {
6202 enabled_intr_status =
6203 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6204 if (intr_status)
6205 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6206 if (enabled_intr_status)
6207 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6208
6209 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6210 }
6211
6212 if (enabled_intr_status && retval == IRQ_NONE) {
6213 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x\n",
6214 __func__, intr_status);
6215 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6216 }
6217
6218 spin_unlock(hba->host->host_lock);
6219 return retval;
6220}
6221
6222static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6223{
6224 int err = 0;
6225 u32 mask = 1 << tag;
6226 unsigned long flags;
6227
6228 if (!test_bit(tag, &hba->outstanding_tasks))
6229 goto out;
6230
6231 spin_lock_irqsave(hba->host->host_lock, flags);
6232 ufshcd_utmrl_clear(hba, tag);
6233 spin_unlock_irqrestore(hba->host->host_lock, flags);
6234
6235
6236 err = ufshcd_wait_for_register(hba,
6237 REG_UTP_TASK_REQ_DOOR_BELL,
6238 mask, 0, 1000, 1000);
6239out:
6240 return err;
6241}
6242
6243static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6244 struct utp_task_req_desc *treq, u8 tm_function)
6245{
6246 struct request_queue *q = hba->tmf_queue;
6247 struct Scsi_Host *host = hba->host;
6248 DECLARE_COMPLETION_ONSTACK(wait);
6249 struct request *req;
6250 unsigned long flags;
6251 int free_slot, task_tag, err;
6252
6253
6254
6255
6256
6257
6258 req = blk_get_request(q, REQ_OP_DRV_OUT, BLK_MQ_REQ_RESERVED);
6259 req->end_io_data = &wait;
6260 free_slot = req->tag;
6261 WARN_ON_ONCE(free_slot < 0 || free_slot >= hba->nutmrs);
6262 ufshcd_hold(hba, false);
6263
6264 spin_lock_irqsave(host->host_lock, flags);
6265 task_tag = hba->nutrs + free_slot;
6266
6267 treq->req_header.dword_0 |= cpu_to_be32(task_tag);
6268
6269 memcpy(hba->utmrdl_base_addr + free_slot, treq, sizeof(*treq));
6270 ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
6271
6272
6273 __set_bit(free_slot, &hba->outstanding_tasks);
6274
6275
6276 wmb();
6277
6278 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
6279
6280 wmb();
6281
6282 spin_unlock_irqrestore(host->host_lock, flags);
6283
6284 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
6285
6286
6287 err = wait_for_completion_io_timeout(&wait,
6288 msecs_to_jiffies(TM_CMD_TIMEOUT));
6289 if (!err) {
6290
6291
6292
6293
6294 req->end_io_data = NULL;
6295 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
6296 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6297 __func__, tm_function);
6298 if (ufshcd_clear_tm_cmd(hba, free_slot))
6299 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
6300 __func__, free_slot);
6301 err = -ETIMEDOUT;
6302 } else {
6303 err = 0;
6304 memcpy(treq, hba->utmrdl_base_addr + free_slot, sizeof(*treq));
6305
6306 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
6307 }
6308
6309 spin_lock_irqsave(hba->host->host_lock, flags);
6310 __clear_bit(free_slot, &hba->outstanding_tasks);
6311 spin_unlock_irqrestore(hba->host->host_lock, flags);
6312
6313 blk_put_request(req);
6314
6315 ufshcd_release(hba);
6316 return err;
6317}
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6330 u8 tm_function, u8 *tm_response)
6331{
6332 struct utp_task_req_desc treq = { { 0 }, };
6333 int ocs_value, err;
6334
6335
6336 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6337 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6338
6339
6340 treq.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6341 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6342 treq.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6343
6344
6345
6346
6347
6348 treq.input_param1 = cpu_to_be32(lun_id);
6349 treq.input_param2 = cpu_to_be32(task_id);
6350
6351 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6352 if (err == -ETIMEDOUT)
6353 return err;
6354
6355 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6356 if (ocs_value != OCS_SUCCESS)
6357 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6358 __func__, ocs_value);
6359 else if (tm_response)
6360 *tm_response = be32_to_cpu(treq.output_param1) &
6361 MASK_TM_SERVICE_RESP;
6362 return err;
6363}
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6383 struct utp_upiu_req *req_upiu,
6384 struct utp_upiu_req *rsp_upiu,
6385 u8 *desc_buff, int *buff_len,
6386 enum dev_cmd_type cmd_type,
6387 enum query_opcode desc_op)
6388{
6389 struct request_queue *q = hba->cmd_queue;
6390 struct request *req;
6391 struct ufshcd_lrb *lrbp;
6392 int err = 0;
6393 int tag;
6394 struct completion wait;
6395 unsigned long flags;
6396 u8 upiu_flags;
6397
6398 down_read(&hba->clk_scaling_lock);
6399
6400 req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
6401 if (IS_ERR(req)) {
6402 err = PTR_ERR(req);
6403 goto out_unlock;
6404 }
6405 tag = req->tag;
6406 WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
6407
6408 init_completion(&wait);
6409 lrbp = &hba->lrb[tag];
6410 WARN_ON(lrbp->cmd);
6411
6412 lrbp->cmd = NULL;
6413 lrbp->sense_bufflen = 0;
6414 lrbp->sense_buffer = NULL;
6415 lrbp->task_tag = tag;
6416 lrbp->lun = 0;
6417 lrbp->intr_cmd = true;
6418 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
6419 hba->dev_cmd.type = cmd_type;
6420
6421 switch (hba->ufs_version) {
6422 case UFSHCI_VERSION_10:
6423 case UFSHCI_VERSION_11:
6424 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6425 break;
6426 default:
6427 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6428 break;
6429 }
6430
6431
6432 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6433
6434 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6435
6436
6437 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6438 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6439
6440
6441
6442
6443 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6444 *buff_len = 0;
6445 }
6446
6447 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6448
6449 hba->dev_cmd.complete = &wait;
6450
6451
6452 wmb();
6453 spin_lock_irqsave(hba->host->host_lock, flags);
6454 ufshcd_send_command(hba, tag);
6455 spin_unlock_irqrestore(hba->host->host_lock, flags);
6456
6457
6458
6459
6460
6461
6462 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6463
6464
6465 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
6466 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6467 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6468 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6469 MASK_QUERY_DATA_SEG_LEN;
6470
6471 if (*buff_len >= resp_len) {
6472 memcpy(desc_buff, descp, resp_len);
6473 *buff_len = resp_len;
6474 } else {
6475 dev_warn(hba->dev,
6476 "%s: rsp size %d is bigger than buffer size %d",
6477 __func__, resp_len, *buff_len);
6478 *buff_len = 0;
6479 err = -EINVAL;
6480 }
6481 }
6482
6483 blk_put_request(req);
6484out_unlock:
6485 up_read(&hba->clk_scaling_lock);
6486 return err;
6487}
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6505 struct utp_upiu_req *req_upiu,
6506 struct utp_upiu_req *rsp_upiu,
6507 int msgcode,
6508 u8 *desc_buff, int *buff_len,
6509 enum query_opcode desc_op)
6510{
6511 int err;
6512 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
6513 struct utp_task_req_desc treq = { { 0 }, };
6514 int ocs_value;
6515 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6516
6517 switch (msgcode) {
6518 case UPIU_TRANSACTION_NOP_OUT:
6519 cmd_type = DEV_CMD_TYPE_NOP;
6520 fallthrough;
6521 case UPIU_TRANSACTION_QUERY_REQ:
6522 ufshcd_hold(hba, false);
6523 mutex_lock(&hba->dev_cmd.lock);
6524 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6525 desc_buff, buff_len,
6526 cmd_type, desc_op);
6527 mutex_unlock(&hba->dev_cmd.lock);
6528 ufshcd_release(hba);
6529
6530 break;
6531 case UPIU_TRANSACTION_TASK_REQ:
6532 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6533 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6534
6535 memcpy(&treq.req_header, req_upiu, sizeof(*req_upiu));
6536
6537 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6538 if (err == -ETIMEDOUT)
6539 break;
6540
6541 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6542 if (ocs_value != OCS_SUCCESS) {
6543 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6544 ocs_value);
6545 break;
6546 }
6547
6548 memcpy(rsp_upiu, &treq.rsp_header, sizeof(*rsp_upiu));
6549
6550 break;
6551 default:
6552 err = -EINVAL;
6553
6554 break;
6555 }
6556
6557 return err;
6558}
6559
6560
6561
6562
6563
6564
6565
6566
6567static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
6568{
6569 struct Scsi_Host *host;
6570 struct ufs_hba *hba;
6571 unsigned int tag;
6572 u32 pos;
6573 int err;
6574 u8 resp = 0xF;
6575 struct ufshcd_lrb *lrbp;
6576 unsigned long flags;
6577
6578 host = cmd->device->host;
6579 hba = shost_priv(host);
6580 tag = cmd->request->tag;
6581
6582 lrbp = &hba->lrb[tag];
6583 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
6584 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6585 if (!err)
6586 err = resp;
6587 goto out;
6588 }
6589
6590
6591 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6592 if (hba->lrb[pos].lun == lrbp->lun) {
6593 err = ufshcd_clear_cmd(hba, pos);
6594 if (err)
6595 break;
6596 }
6597 }
6598 spin_lock_irqsave(host->host_lock, flags);
6599 ufshcd_transfer_req_compl(hba);
6600 spin_unlock_irqrestore(host->host_lock, flags);
6601
6602out:
6603 hba->req_abort_count = 0;
6604 ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
6605 if (!err) {
6606 err = SUCCESS;
6607 } else {
6608 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6609 err = FAILED;
6610 }
6611 return err;
6612}
6613
6614static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6615{
6616 struct ufshcd_lrb *lrbp;
6617 int tag;
6618
6619 for_each_set_bit(tag, &bitmap, hba->nutrs) {
6620 lrbp = &hba->lrb[tag];
6621 lrbp->req_abort_skip = true;
6622 }
6623}
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6638{
6639 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6640 int err = 0;
6641 int poll_cnt;
6642 u8 resp = 0xF;
6643 u32 reg;
6644
6645 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6646 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6647 UFS_QUERY_TASK, &resp);
6648 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6649
6650 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6651 __func__, tag);
6652 break;
6653 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6654
6655
6656
6657
6658 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6659 __func__, tag);
6660 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6661 if (reg & (1 << tag)) {
6662
6663 usleep_range(100, 200);
6664 continue;
6665 }
6666
6667 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6668 __func__, tag);
6669 goto out;
6670 } else {
6671 dev_err(hba->dev,
6672 "%s: no response from device. tag = %d, err %d\n",
6673 __func__, tag, err);
6674 if (!err)
6675 err = resp;
6676 goto out;
6677 }
6678 }
6679
6680 if (!poll_cnt) {
6681 err = -EBUSY;
6682 goto out;
6683 }
6684
6685 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6686 UFS_ABORT_TASK, &resp);
6687 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6688 if (!err) {
6689 err = resp;
6690 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6691 __func__, tag, err);
6692 }
6693 goto out;
6694 }
6695
6696 err = ufshcd_clear_cmd(hba, tag);
6697 if (err)
6698 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6699 __func__, tag, err);
6700
6701out:
6702 return err;
6703}
6704
6705
6706
6707
6708
6709
6710
6711static int ufshcd_abort(struct scsi_cmnd *cmd)
6712{
6713 struct Scsi_Host *host;
6714 struct ufs_hba *hba;
6715 unsigned long flags;
6716 unsigned int tag;
6717 int err = 0;
6718 struct ufshcd_lrb *lrbp;
6719 u32 reg;
6720
6721 host = cmd->device->host;
6722 hba = shost_priv(host);
6723 tag = cmd->request->tag;
6724 lrbp = &hba->lrb[tag];
6725 if (!ufshcd_valid_tag(hba, tag)) {
6726 dev_err(hba->dev,
6727 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6728 __func__, tag, cmd, cmd->request);
6729 BUG();
6730 }
6731
6732
6733
6734
6735
6736
6737
6738
6739 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
6740 return ufshcd_eh_host_reset_handler(cmd);
6741
6742 ufshcd_hold(hba, false);
6743 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6744
6745 if (!(test_bit(tag, &hba->outstanding_reqs))) {
6746 dev_err(hba->dev,
6747 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
6748 __func__, tag, hba->outstanding_reqs, reg);
6749 goto out;
6750 }
6751
6752
6753 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
6754
6755
6756
6757
6758
6759
6760
6761
6762 scsi_print_command(hba->lrb[tag].cmd);
6763 if (!hba->req_abort_count) {
6764 ufshcd_update_reg_hist(&hba->ufs_stats.task_abort, 0);
6765 ufshcd_print_host_regs(hba);
6766 ufshcd_print_host_state(hba);
6767 ufshcd_print_pwr_info(hba);
6768 ufshcd_print_trs(hba, 1 << tag, true);
6769 } else {
6770 ufshcd_print_trs(hba, 1 << tag, false);
6771 }
6772 hba->req_abort_count++;
6773
6774 if (!(reg & (1 << tag))) {
6775 dev_err(hba->dev,
6776 "%s: cmd was completed, but without a notifying intr, tag = %d",
6777 __func__, tag);
6778 goto cleanup;
6779 }
6780
6781
6782 if (lrbp->req_abort_skip)
6783 err = -EIO;
6784 else
6785 err = ufshcd_try_to_abort_task(hba, tag);
6786
6787 if (!err) {
6788cleanup:
6789 spin_lock_irqsave(host->host_lock, flags);
6790 __ufshcd_transfer_req_compl(hba, (1UL << tag));
6791 spin_unlock_irqrestore(host->host_lock, flags);
6792out:
6793 err = SUCCESS;
6794 } else {
6795 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6796 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
6797 err = FAILED;
6798 }
6799
6800
6801
6802
6803
6804 ufshcd_release(hba);
6805 return err;
6806}
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6819{
6820 int err;
6821 unsigned long flags;
6822
6823
6824
6825
6826
6827 ufshcd_hba_stop(hba);
6828
6829 spin_lock_irqsave(hba->host->host_lock, flags);
6830 hba->silence_err_logs = true;
6831 ufshcd_complete_requests(hba);
6832 hba->silence_err_logs = false;
6833 spin_unlock_irqrestore(hba->host->host_lock, flags);
6834
6835
6836 ufshcd_set_clk_freq(hba, true);
6837
6838 err = ufshcd_hba_enable(hba);
6839 if (err)
6840 goto out;
6841
6842
6843 err = ufshcd_probe_hba(hba, false);
6844
6845out:
6846 if (err)
6847 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
6848 ufshcd_update_reg_hist(&hba->ufs_stats.host_reset, (u32)err);
6849 return err;
6850}
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6862{
6863 u32 saved_err;
6864 u32 saved_uic_err;
6865 int err = 0;
6866 unsigned long flags;
6867 int retries = MAX_HOST_RESET_RETRIES;
6868
6869
6870
6871
6872
6873 spin_lock_irqsave(hba->host->host_lock, flags);
6874 saved_err = hba->saved_err;
6875 saved_uic_err = hba->saved_uic_err;
6876 hba->saved_err = 0;
6877 hba->saved_uic_err = 0;
6878 spin_unlock_irqrestore(hba->host->host_lock, flags);
6879
6880 do {
6881
6882 ufshcd_vops_device_reset(hba);
6883
6884 err = ufshcd_host_reset_and_restore(hba);
6885 } while (err && --retries);
6886
6887 spin_lock_irqsave(hba->host->host_lock, flags);
6888
6889
6890
6891
6892 scsi_report_bus_reset(hba->host, 0);
6893 if (err) {
6894 hba->saved_err |= saved_err;
6895 hba->saved_uic_err |= saved_uic_err;
6896 }
6897 spin_unlock_irqrestore(hba->host->host_lock, flags);
6898
6899 return err;
6900}
6901
6902
6903
6904
6905
6906
6907
6908static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6909{
6910 int err = SUCCESS;
6911 unsigned long flags;
6912 struct ufs_hba *hba;
6913
6914 hba = shost_priv(cmd->device->host);
6915
6916 spin_lock_irqsave(hba->host->host_lock, flags);
6917 hba->force_reset = true;
6918 ufshcd_schedule_eh_work(hba);
6919 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
6920 spin_unlock_irqrestore(hba->host->host_lock, flags);
6921
6922 flush_work(&hba->eh_work);
6923
6924 spin_lock_irqsave(hba->host->host_lock, flags);
6925 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
6926 err = FAILED;
6927 spin_unlock_irqrestore(hba->host->host_lock, flags);
6928
6929 return err;
6930}
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6941{
6942 int i;
6943 int curr_uA;
6944 u16 data;
6945 u16 unit;
6946
6947 for (i = start_scan; i >= 0; i--) {
6948 data = be16_to_cpup((__be16 *)&buff[2 * i]);
6949 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6950 ATTR_ICC_LVL_UNIT_OFFSET;
6951 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6952 switch (unit) {
6953 case UFSHCD_NANO_AMP:
6954 curr_uA = curr_uA / 1000;
6955 break;
6956 case UFSHCD_MILI_AMP:
6957 curr_uA = curr_uA * 1000;
6958 break;
6959 case UFSHCD_AMP:
6960 curr_uA = curr_uA * 1000 * 1000;
6961 break;
6962 case UFSHCD_MICRO_AMP:
6963 default:
6964 break;
6965 }
6966 if (sup_curr_uA >= curr_uA)
6967 break;
6968 }
6969 if (i < 0) {
6970 i = 0;
6971 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6972 }
6973
6974 return (u32)i;
6975}
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6987 u8 *desc_buf, int len)
6988{
6989 u32 icc_level = 0;
6990
6991 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6992 !hba->vreg_info.vccq2) {
6993 dev_err(hba->dev,
6994 "%s: Regulator capability was not set, actvIccLevel=%d",
6995 __func__, icc_level);
6996 goto out;
6997 }
6998
6999 if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
7000 icc_level = ufshcd_get_max_icc_level(
7001 hba->vreg_info.vcc->max_uA,
7002 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7003 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7004
7005 if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
7006 icc_level = ufshcd_get_max_icc_level(
7007 hba->vreg_info.vccq->max_uA,
7008 icc_level,
7009 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7010
7011 if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
7012 icc_level = ufshcd_get_max_icc_level(
7013 hba->vreg_info.vccq2->max_uA,
7014 icc_level,
7015 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7016out:
7017 return icc_level;
7018}
7019
7020static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7021{
7022 int ret;
7023 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
7024 u8 *desc_buf;
7025 u32 icc_level;
7026
7027 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7028 if (!desc_buf)
7029 return;
7030
7031 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7032 desc_buf, buff_len);
7033 if (ret) {
7034 dev_err(hba->dev,
7035 "%s: Failed reading power descriptor.len = %d ret = %d",
7036 __func__, buff_len, ret);
7037 goto out;
7038 }
7039
7040 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7041 buff_len);
7042 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7043
7044 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7045 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7046
7047 if (ret)
7048 dev_err(hba->dev,
7049 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7050 __func__, icc_level, ret);
7051
7052out:
7053 kfree(desc_buf);
7054}
7055
7056static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7057{
7058 scsi_autopm_get_device(sdev);
7059 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7060 if (sdev->rpm_autosuspend)
7061 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7062 RPM_AUTOSUSPEND_DELAY_MS);
7063 scsi_autopm_put_device(sdev);
7064}
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7093{
7094 int ret = 0;
7095 struct scsi_device *sdev_rpmb;
7096 struct scsi_device *sdev_boot;
7097
7098 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7099 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7100 if (IS_ERR(hba->sdev_ufs_device)) {
7101 ret = PTR_ERR(hba->sdev_ufs_device);
7102 hba->sdev_ufs_device = NULL;
7103 goto out;
7104 }
7105 ufshcd_blk_pm_runtime_init(hba->sdev_ufs_device);
7106 scsi_device_put(hba->sdev_ufs_device);
7107
7108 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7109 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7110 if (IS_ERR(sdev_rpmb)) {
7111 ret = PTR_ERR(sdev_rpmb);
7112 goto remove_sdev_ufs_device;
7113 }
7114 ufshcd_blk_pm_runtime_init(sdev_rpmb);
7115 scsi_device_put(sdev_rpmb);
7116
7117 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7118 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
7119 if (IS_ERR(sdev_boot)) {
7120 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
7121 } else {
7122 ufshcd_blk_pm_runtime_init(sdev_boot);
7123 scsi_device_put(sdev_boot);
7124 }
7125 goto out;
7126
7127remove_sdev_ufs_device:
7128 scsi_remove_device(hba->sdev_ufs_device);
7129out:
7130 return ret;
7131}
7132
7133static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7134{
7135 struct ufs_dev_info *dev_info = &hba->dev_info;
7136 u8 lun;
7137 u32 d_lu_wb_buf_alloc;
7138
7139 if (!ufshcd_is_wb_allowed(hba))
7140 return;
7141
7142
7143
7144
7145
7146 if (!(dev_info->wspecversion >= 0x310 ||
7147 dev_info->wspecversion == 0x220 ||
7148 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7149 goto wb_disabled;
7150
7151 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7152 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
7153 goto wb_disabled;
7154
7155 dev_info->d_ext_ufs_feature_sup =
7156 get_unaligned_be32(desc_buf +
7157 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7158
7159 if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
7160 goto wb_disabled;
7161
7162
7163
7164
7165
7166
7167
7168 dev_info->b_wb_buffer_type =
7169 desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7170
7171 dev_info->b_presrv_uspc_en =
7172 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7173
7174 if (dev_info->b_wb_buffer_type == WB_BUF_MODE_SHARED) {
7175 dev_info->d_wb_alloc_units =
7176 get_unaligned_be32(desc_buf +
7177 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
7178 if (!dev_info->d_wb_alloc_units)
7179 goto wb_disabled;
7180 } else {
7181 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7182 d_lu_wb_buf_alloc = 0;
7183 ufshcd_read_unit_desc_param(hba,
7184 lun,
7185 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7186 (u8 *)&d_lu_wb_buf_alloc,
7187 sizeof(d_lu_wb_buf_alloc));
7188 if (d_lu_wb_buf_alloc) {
7189 dev_info->wb_dedicated_lu = lun;
7190 break;
7191 }
7192 }
7193
7194 if (!d_lu_wb_buf_alloc)
7195 goto wb_disabled;
7196 }
7197 return;
7198
7199wb_disabled:
7200 hba->caps &= ~UFSHCD_CAP_WB_EN;
7201}
7202
7203void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
7204{
7205 struct ufs_dev_fix *f;
7206 struct ufs_dev_info *dev_info = &hba->dev_info;
7207
7208 if (!fixups)
7209 return;
7210
7211 for (f = fixups; f->quirk; f++) {
7212 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7213 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7214 ((dev_info->model &&
7215 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7216 !strcmp(f->model, UFS_ANY_MODEL)))
7217 hba->dev_quirks |= f->quirk;
7218 }
7219}
7220EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
7221
7222static void ufs_fixup_device_setup(struct ufs_hba *hba)
7223{
7224
7225 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
7226
7227
7228 ufshcd_vops_fixup_dev_quirks(hba);
7229}
7230
7231static int ufs_get_device_desc(struct ufs_hba *hba)
7232{
7233 int err;
7234 u8 model_index;
7235 u8 *desc_buf;
7236 struct ufs_dev_info *dev_info = &hba->dev_info;
7237
7238 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7239 if (!desc_buf) {
7240 err = -ENOMEM;
7241 goto out;
7242 }
7243
7244 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7245 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
7246 if (err) {
7247 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7248 __func__, err);
7249 goto out;
7250 }
7251
7252
7253
7254
7255
7256 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
7257 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7258
7259
7260 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7261 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7262
7263 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
7264
7265 err = ufshcd_read_string_desc(hba, model_index,
7266 &dev_info->model, SD_ASCII_STD);
7267 if (err < 0) {
7268 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7269 __func__, err);
7270 goto out;
7271 }
7272
7273 ufs_fixup_device_setup(hba);
7274
7275 ufshcd_wb_probe(hba, desc_buf);
7276
7277
7278
7279
7280
7281 err = 0;
7282
7283out:
7284 kfree(desc_buf);
7285 return err;
7286}
7287
7288static void ufs_put_device_desc(struct ufs_hba *hba)
7289{
7290 struct ufs_dev_info *dev_info = &hba->dev_info;
7291
7292 kfree(dev_info->model);
7293 dev_info->model = NULL;
7294}
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7308{
7309 int ret = 0;
7310 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7311
7312 ret = ufshcd_dme_peer_get(hba,
7313 UIC_ARG_MIB_SEL(
7314 RX_MIN_ACTIVATETIME_CAPABILITY,
7315 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7316 &peer_rx_min_activatetime);
7317 if (ret)
7318 goto out;
7319
7320
7321 tuned_pa_tactivate =
7322 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7323 / PA_TACTIVATE_TIME_UNIT_US);
7324 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7325 tuned_pa_tactivate);
7326
7327out:
7328 return ret;
7329}
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7343{
7344 int ret = 0;
7345 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7346 u32 max_hibern8_time, tuned_pa_hibern8time;
7347
7348 ret = ufshcd_dme_get(hba,
7349 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7350 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7351 &local_tx_hibern8_time_cap);
7352 if (ret)
7353 goto out;
7354
7355 ret = ufshcd_dme_peer_get(hba,
7356 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7357 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7358 &peer_rx_hibern8_time_cap);
7359 if (ret)
7360 goto out;
7361
7362 max_hibern8_time = max(local_tx_hibern8_time_cap,
7363 peer_rx_hibern8_time_cap);
7364
7365 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7366 / PA_HIBERN8_TIME_UNIT_US);
7367 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7368 tuned_pa_hibern8time);
7369out:
7370 return ret;
7371}
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7385{
7386 int ret = 0;
7387 u32 granularity, peer_granularity;
7388 u32 pa_tactivate, peer_pa_tactivate;
7389 u32 pa_tactivate_us, peer_pa_tactivate_us;
7390 u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7391
7392 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7393 &granularity);
7394 if (ret)
7395 goto out;
7396
7397 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7398 &peer_granularity);
7399 if (ret)
7400 goto out;
7401
7402 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7403 (granularity > PA_GRANULARITY_MAX_VAL)) {
7404 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7405 __func__, granularity);
7406 return -EINVAL;
7407 }
7408
7409 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7410 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7411 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7412 __func__, peer_granularity);
7413 return -EINVAL;
7414 }
7415
7416 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7417 if (ret)
7418 goto out;
7419
7420 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7421 &peer_pa_tactivate);
7422 if (ret)
7423 goto out;
7424
7425 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7426 peer_pa_tactivate_us = peer_pa_tactivate *
7427 gran_to_us_table[peer_granularity - 1];
7428
7429 if (pa_tactivate_us > peer_pa_tactivate_us) {
7430 u32 new_peer_pa_tactivate;
7431
7432 new_peer_pa_tactivate = pa_tactivate_us /
7433 gran_to_us_table[peer_granularity - 1];
7434 new_peer_pa_tactivate++;
7435 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7436 new_peer_pa_tactivate);
7437 }
7438
7439out:
7440 return ret;
7441}
7442
7443static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
7444{
7445 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7446 ufshcd_tune_pa_tactivate(hba);
7447 ufshcd_tune_pa_hibern8time(hba);
7448 }
7449
7450 ufshcd_vops_apply_dev_quirks(hba);
7451
7452 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7453
7454 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
7455
7456 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7457 ufshcd_quirk_tune_host_pa_tactivate(hba);
7458}
7459
7460static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7461{
7462 hba->ufs_stats.hibern8_exit_cnt = 0;
7463 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7464 hba->req_abort_count = 0;
7465}
7466
7467static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7468{
7469 int err;
7470 size_t buff_len;
7471 u8 *desc_buf;
7472
7473 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
7474 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7475 if (!desc_buf) {
7476 err = -ENOMEM;
7477 goto out;
7478 }
7479
7480 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7481 desc_buf, buff_len);
7482 if (err) {
7483 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7484 __func__, err);
7485 goto out;
7486 }
7487
7488 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7489 hba->dev_info.max_lu_supported = 32;
7490 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7491 hba->dev_info.max_lu_supported = 8;
7492
7493out:
7494 kfree(desc_buf);
7495 return err;
7496}
7497
7498static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7499 {19200000, REF_CLK_FREQ_19_2_MHZ},
7500 {26000000, REF_CLK_FREQ_26_MHZ},
7501 {38400000, REF_CLK_FREQ_38_4_MHZ},
7502 {52000000, REF_CLK_FREQ_52_MHZ},
7503 {0, REF_CLK_FREQ_INVAL},
7504};
7505
7506static enum ufs_ref_clk_freq
7507ufs_get_bref_clk_from_hz(unsigned long freq)
7508{
7509 int i;
7510
7511 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7512 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7513 return ufs_ref_clk_freqs[i].val;
7514
7515 return REF_CLK_FREQ_INVAL;
7516}
7517
7518void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7519{
7520 unsigned long freq;
7521
7522 freq = clk_get_rate(refclk);
7523
7524 hba->dev_ref_clk_freq =
7525 ufs_get_bref_clk_from_hz(freq);
7526
7527 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7528 dev_err(hba->dev,
7529 "invalid ref_clk setting = %ld\n", freq);
7530}
7531
7532static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7533{
7534 int err;
7535 u32 ref_clk;
7536 u32 freq = hba->dev_ref_clk_freq;
7537
7538 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7539 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7540
7541 if (err) {
7542 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7543 err);
7544 goto out;
7545 }
7546
7547 if (ref_clk == freq)
7548 goto out;
7549
7550 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7551 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7552
7553 if (err) {
7554 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7555 ufs_ref_clk_freqs[freq].freq_hz);
7556 goto out;
7557 }
7558
7559 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7560 ufs_ref_clk_freqs[freq].freq_hz);
7561
7562out:
7563 return err;
7564}
7565
7566static int ufshcd_device_params_init(struct ufs_hba *hba)
7567{
7568 bool flag;
7569 int ret, i;
7570
7571
7572 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7573 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
7574
7575
7576 ret = ufshcd_device_geo_params_init(hba);
7577 if (ret)
7578 goto out;
7579
7580
7581 ret = ufs_get_device_desc(hba);
7582 if (ret) {
7583 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7584 __func__, ret);
7585 goto out;
7586 }
7587
7588 ufshcd_get_ref_clk_gating_wait(hba);
7589
7590 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
7591 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
7592 hba->dev_info.f_power_on_wp_en = flag;
7593
7594
7595 if (ufshcd_get_max_pwr_mode(hba))
7596 dev_err(hba->dev,
7597 "%s: Failed getting max supported power mode\n",
7598 __func__);
7599out:
7600 return ret;
7601}
7602
7603
7604
7605
7606
7607static int ufshcd_add_lus(struct ufs_hba *hba)
7608{
7609 int ret;
7610
7611
7612 ret = ufshcd_scsi_add_wlus(hba);
7613 if (ret)
7614 goto out;
7615
7616
7617 if (ufshcd_is_clkscaling_supported(hba)) {
7618 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7619 &hba->pwr_info,
7620 sizeof(struct ufs_pa_layer_attr));
7621 hba->clk_scaling.saved_pwr_info.is_valid = true;
7622 if (!hba->devfreq) {
7623 ret = ufshcd_devfreq_init(hba);
7624 if (ret)
7625 goto out;
7626 }
7627
7628 hba->clk_scaling.is_allowed = true;
7629 }
7630
7631 ufs_bsg_probe(hba);
7632 scsi_scan_host(hba->host);
7633 pm_runtime_put_sync(hba->dev);
7634
7635out:
7636 return ret;
7637}
7638
7639
7640
7641
7642
7643
7644
7645
7646static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
7647{
7648 int ret;
7649 unsigned long flags;
7650 ktime_t start = ktime_get();
7651
7652 ret = ufshcd_link_startup(hba);
7653 if (ret)
7654 goto out;
7655
7656
7657 ufshcd_clear_dbg_ufs_stats(hba);
7658
7659
7660 ufshcd_set_link_active(hba);
7661
7662
7663 ret = ufshcd_verify_dev_init(hba);
7664 if (ret)
7665 goto out;
7666
7667
7668 ret = ufshcd_complete_dev_init(hba);
7669 if (ret)
7670 goto out;
7671
7672
7673
7674
7675
7676 if (async) {
7677 ret = ufshcd_device_params_init(hba);
7678 if (ret)
7679 goto out;
7680 }
7681
7682 ufshcd_tune_unipro_params(hba);
7683
7684
7685 ufshcd_set_ufs_dev_active(hba);
7686 ufshcd_force_reset_auto_bkops(hba);
7687 hba->wlun_dev_clr_ua = true;
7688
7689
7690 if (hba->max_pwr_info.is_valid) {
7691
7692
7693
7694
7695 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
7696 ufshcd_set_dev_ref_clk(hba);
7697 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
7698 if (ret) {
7699 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
7700 __func__, ret);
7701 goto out;
7702 }
7703 ufshcd_print_pwr_info(hba);
7704 }
7705
7706
7707
7708
7709
7710
7711
7712 ufshcd_set_active_icc_lvl(hba);
7713
7714 ufshcd_wb_config(hba);
7715
7716 ufshcd_auto_hibern8_enable(hba);
7717
7718out:
7719 spin_lock_irqsave(hba->host->host_lock, flags);
7720 if (ret)
7721 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7722 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7723 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7724 spin_unlock_irqrestore(hba->host->host_lock, flags);
7725
7726 trace_ufshcd_init(dev_name(hba->dev), ret,
7727 ktime_to_us(ktime_sub(ktime_get(), start)),
7728 hba->curr_dev_pwr_mode, hba->uic_link_state);
7729 return ret;
7730}
7731
7732
7733
7734
7735
7736
7737static void ufshcd_async_scan(void *data, async_cookie_t cookie)
7738{
7739 struct ufs_hba *hba = (struct ufs_hba *)data;
7740 int ret;
7741
7742
7743 ret = ufshcd_probe_hba(hba, true);
7744 if (ret)
7745 goto out;
7746
7747
7748 ret = ufshcd_add_lus(hba);
7749out:
7750
7751
7752
7753
7754 if (ret) {
7755 pm_runtime_put_sync(hba->dev);
7756 ufshcd_exit_clk_scaling(hba);
7757 ufshcd_hba_exit(hba);
7758 }
7759}
7760
7761static const struct attribute_group *ufshcd_driver_groups[] = {
7762 &ufs_sysfs_unit_descriptor_group,
7763 &ufs_sysfs_lun_attributes_group,
7764 NULL,
7765};
7766
7767static struct ufs_hba_variant_params ufs_hba_vps = {
7768 .hba_enable_delay_us = 1000,
7769 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
7770 .devfreq_profile.polling_ms = 100,
7771 .devfreq_profile.target = ufshcd_devfreq_target,
7772 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
7773 .ondemand_data.upthreshold = 70,
7774 .ondemand_data.downdifferential = 5,
7775};
7776
7777static struct scsi_host_template ufshcd_driver_template = {
7778 .module = THIS_MODULE,
7779 .name = UFSHCD,
7780 .proc_name = UFSHCD,
7781 .queuecommand = ufshcd_queuecommand,
7782 .slave_alloc = ufshcd_slave_alloc,
7783 .slave_configure = ufshcd_slave_configure,
7784 .slave_destroy = ufshcd_slave_destroy,
7785 .change_queue_depth = ufshcd_change_queue_depth,
7786 .eh_abort_handler = ufshcd_abort,
7787 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
7788 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
7789 .this_id = -1,
7790 .sg_tablesize = SG_ALL,
7791 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
7792 .can_queue = UFSHCD_CAN_QUEUE,
7793 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
7794 .max_host_blocked = 1,
7795 .track_queue_depth = 1,
7796 .sdev_groups = ufshcd_driver_groups,
7797 .dma_boundary = PAGE_SIZE - 1,
7798 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
7799};
7800
7801static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
7802 int ua)
7803{
7804 int ret;
7805
7806 if (!vreg)
7807 return 0;
7808
7809
7810
7811
7812
7813
7814
7815 if (!vreg->max_uA)
7816 return 0;
7817
7818 ret = regulator_set_load(vreg->reg, ua);
7819 if (ret < 0) {
7820 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
7821 __func__, vreg->name, ua, ret);
7822 }
7823
7824 return ret;
7825}
7826
7827static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
7828 struct ufs_vreg *vreg)
7829{
7830 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
7831}
7832
7833static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
7834 struct ufs_vreg *vreg)
7835{
7836 if (!vreg)
7837 return 0;
7838
7839 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
7840}
7841
7842static int ufshcd_config_vreg(struct device *dev,
7843 struct ufs_vreg *vreg, bool on)
7844{
7845 int ret = 0;
7846 struct regulator *reg;
7847 const char *name;
7848 int min_uV, uA_load;
7849
7850 BUG_ON(!vreg);
7851
7852 reg = vreg->reg;
7853 name = vreg->name;
7854
7855 if (regulator_count_voltages(reg) > 0) {
7856 uA_load = on ? vreg->max_uA : 0;
7857 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
7858 if (ret)
7859 goto out;
7860
7861 if (vreg->min_uV && vreg->max_uV) {
7862 min_uV = on ? vreg->min_uV : 0;
7863 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
7864 if (ret)
7865 dev_err(dev,
7866 "%s: %s set voltage failed, err=%d\n",
7867 __func__, name, ret);
7868 }
7869 }
7870out:
7871 return ret;
7872}
7873
7874static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
7875{
7876 int ret = 0;
7877
7878 if (!vreg || vreg->enabled)
7879 goto out;
7880
7881 ret = ufshcd_config_vreg(dev, vreg, true);
7882 if (!ret)
7883 ret = regulator_enable(vreg->reg);
7884
7885 if (!ret)
7886 vreg->enabled = true;
7887 else
7888 dev_err(dev, "%s: %s enable failed, err=%d\n",
7889 __func__, vreg->name, ret);
7890out:
7891 return ret;
7892}
7893
7894static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
7895{
7896 int ret = 0;
7897
7898 if (!vreg || !vreg->enabled)
7899 goto out;
7900
7901 ret = regulator_disable(vreg->reg);
7902
7903 if (!ret) {
7904
7905 ufshcd_config_vreg(dev, vreg, false);
7906 vreg->enabled = false;
7907 } else {
7908 dev_err(dev, "%s: %s disable failed, err=%d\n",
7909 __func__, vreg->name, ret);
7910 }
7911out:
7912 return ret;
7913}
7914
7915static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
7916{
7917 int ret = 0;
7918 struct device *dev = hba->dev;
7919 struct ufs_vreg_info *info = &hba->vreg_info;
7920
7921 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
7922 if (ret)
7923 goto out;
7924
7925 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
7926 if (ret)
7927 goto out;
7928
7929 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
7930
7931out:
7932 if (ret) {
7933 ufshcd_toggle_vreg(dev, info->vccq2, false);
7934 ufshcd_toggle_vreg(dev, info->vccq, false);
7935 ufshcd_toggle_vreg(dev, info->vcc, false);
7936 }
7937 return ret;
7938}
7939
7940static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
7941{
7942 struct ufs_vreg_info *info = &hba->vreg_info;
7943
7944 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
7945}
7946
7947static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
7948{
7949 int ret = 0;
7950
7951 if (!vreg)
7952 goto out;
7953
7954 vreg->reg = devm_regulator_get(dev, vreg->name);
7955 if (IS_ERR(vreg->reg)) {
7956 ret = PTR_ERR(vreg->reg);
7957 dev_err(dev, "%s: %s get failed, err=%d\n",
7958 __func__, vreg->name, ret);
7959 }
7960out:
7961 return ret;
7962}
7963
7964static int ufshcd_init_vreg(struct ufs_hba *hba)
7965{
7966 int ret = 0;
7967 struct device *dev = hba->dev;
7968 struct ufs_vreg_info *info = &hba->vreg_info;
7969
7970 ret = ufshcd_get_vreg(dev, info->vcc);
7971 if (ret)
7972 goto out;
7973
7974 ret = ufshcd_get_vreg(dev, info->vccq);
7975 if (!ret)
7976 ret = ufshcd_get_vreg(dev, info->vccq2);
7977out:
7978 return ret;
7979}
7980
7981static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7982{
7983 struct ufs_vreg_info *info = &hba->vreg_info;
7984
7985 if (info)
7986 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7987
7988 return 0;
7989}
7990
7991static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7992 bool skip_ref_clk)
7993{
7994 int ret = 0;
7995 struct ufs_clk_info *clki;
7996 struct list_head *head = &hba->clk_list_head;
7997 unsigned long flags;
7998 ktime_t start = ktime_get();
7999 bool clk_state_changed = false;
8000
8001 if (list_empty(head))
8002 goto out;
8003
8004 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8005 if (ret)
8006 return ret;
8007
8008 list_for_each_entry(clki, head, list) {
8009 if (!IS_ERR_OR_NULL(clki->clk)) {
8010 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
8011 continue;
8012
8013 clk_state_changed = on ^ clki->enabled;
8014 if (on && !clki->enabled) {
8015 ret = clk_prepare_enable(clki->clk);
8016 if (ret) {
8017 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8018 __func__, clki->name, ret);
8019 goto out;
8020 }
8021 } else if (!on && clki->enabled) {
8022 clk_disable_unprepare(clki->clk);
8023 }
8024 clki->enabled = on;
8025 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8026 clki->name, on ? "en" : "dis");
8027 }
8028 }
8029
8030 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8031 if (ret)
8032 return ret;
8033
8034out:
8035 if (ret) {
8036 list_for_each_entry(clki, head, list) {
8037 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8038 clk_disable_unprepare(clki->clk);
8039 }
8040 } else if (!ret && on) {
8041 spin_lock_irqsave(hba->host->host_lock, flags);
8042 hba->clk_gating.state = CLKS_ON;
8043 trace_ufshcd_clk_gating(dev_name(hba->dev),
8044 hba->clk_gating.state);
8045 spin_unlock_irqrestore(hba->host->host_lock, flags);
8046 }
8047
8048 if (clk_state_changed)
8049 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8050 (on ? "on" : "off"),
8051 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
8052 return ret;
8053}
8054
8055static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8056{
8057 return __ufshcd_setup_clocks(hba, on, false);
8058}
8059
8060static int ufshcd_init_clocks(struct ufs_hba *hba)
8061{
8062 int ret = 0;
8063 struct ufs_clk_info *clki;
8064 struct device *dev = hba->dev;
8065 struct list_head *head = &hba->clk_list_head;
8066
8067 if (list_empty(head))
8068 goto out;
8069
8070 list_for_each_entry(clki, head, list) {
8071 if (!clki->name)
8072 continue;
8073
8074 clki->clk = devm_clk_get(dev, clki->name);
8075 if (IS_ERR(clki->clk)) {
8076 ret = PTR_ERR(clki->clk);
8077 dev_err(dev, "%s: %s clk get failed, %d\n",
8078 __func__, clki->name, ret);
8079 goto out;
8080 }
8081
8082
8083
8084
8085
8086
8087 if (!strcmp(clki->name, "ref_clk"))
8088 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8089
8090 if (clki->max_freq) {
8091 ret = clk_set_rate(clki->clk, clki->max_freq);
8092 if (ret) {
8093 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8094 __func__, clki->name,
8095 clki->max_freq, ret);
8096 goto out;
8097 }
8098 clki->curr_freq = clki->max_freq;
8099 }
8100 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8101 clki->name, clk_get_rate(clki->clk));
8102 }
8103out:
8104 return ret;
8105}
8106
8107static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8108{
8109 int err = 0;
8110
8111 if (!hba->vops)
8112 goto out;
8113
8114 err = ufshcd_vops_init(hba);
8115 if (err)
8116 goto out;
8117
8118 err = ufshcd_vops_setup_regulators(hba, true);
8119 if (err)
8120 ufshcd_vops_exit(hba);
8121out:
8122 if (err)
8123 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
8124 __func__, ufshcd_get_var_name(hba), err);
8125 return err;
8126}
8127
8128static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8129{
8130 if (!hba->vops)
8131 return;
8132
8133 ufshcd_vops_setup_regulators(hba, false);
8134
8135 ufshcd_vops_exit(hba);
8136}
8137
8138static int ufshcd_hba_init(struct ufs_hba *hba)
8139{
8140 int err;
8141
8142
8143
8144
8145
8146
8147
8148
8149 err = ufshcd_init_hba_vreg(hba);
8150 if (err)
8151 goto out;
8152
8153 err = ufshcd_setup_hba_vreg(hba, true);
8154 if (err)
8155 goto out;
8156
8157 err = ufshcd_init_clocks(hba);
8158 if (err)
8159 goto out_disable_hba_vreg;
8160
8161 err = ufshcd_setup_clocks(hba, true);
8162 if (err)
8163 goto out_disable_hba_vreg;
8164
8165 err = ufshcd_init_vreg(hba);
8166 if (err)
8167 goto out_disable_clks;
8168
8169 err = ufshcd_setup_vreg(hba, true);
8170 if (err)
8171 goto out_disable_clks;
8172
8173 err = ufshcd_variant_hba_init(hba);
8174 if (err)
8175 goto out_disable_vreg;
8176
8177 hba->is_powered = true;
8178 goto out;
8179
8180out_disable_vreg:
8181 ufshcd_setup_vreg(hba, false);
8182out_disable_clks:
8183 ufshcd_setup_clocks(hba, false);
8184out_disable_hba_vreg:
8185 ufshcd_setup_hba_vreg(hba, false);
8186out:
8187 return err;
8188}
8189
8190static void ufshcd_hba_exit(struct ufs_hba *hba)
8191{
8192 if (hba->is_powered) {
8193 ufshcd_variant_hba_exit(hba);
8194 ufshcd_setup_vreg(hba, false);
8195 ufshcd_suspend_clkscaling(hba);
8196 if (ufshcd_is_clkscaling_supported(hba))
8197 if (hba->devfreq)
8198 ufshcd_suspend_clkscaling(hba);
8199 ufshcd_setup_clocks(hba, false);
8200 ufshcd_setup_hba_vreg(hba, false);
8201 hba->is_powered = false;
8202 ufs_put_device_desc(hba);
8203 }
8204}
8205
8206static int
8207ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
8208{
8209 unsigned char cmd[6] = {REQUEST_SENSE,
8210 0,
8211 0,
8212 0,
8213 UFS_SENSE_SIZE,
8214 0};
8215 char *buffer;
8216 int ret;
8217
8218 buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
8219 if (!buffer) {
8220 ret = -ENOMEM;
8221 goto out;
8222 }
8223
8224 ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
8225 UFS_SENSE_SIZE, NULL, NULL,
8226 msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
8227 if (ret)
8228 pr_err("%s: failed with err %d\n", __func__, ret);
8229
8230 kfree(buffer);
8231out:
8232 return ret;
8233}
8234
8235
8236
8237
8238
8239
8240
8241
8242
8243
8244static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8245 enum ufs_dev_pwr_mode pwr_mode)
8246{
8247 unsigned char cmd[6] = { START_STOP };
8248 struct scsi_sense_hdr sshdr;
8249 struct scsi_device *sdp;
8250 unsigned long flags;
8251 int ret;
8252
8253 spin_lock_irqsave(hba->host->host_lock, flags);
8254 sdp = hba->sdev_ufs_device;
8255 if (sdp) {
8256 ret = scsi_device_get(sdp);
8257 if (!ret && !scsi_device_online(sdp)) {
8258 ret = -ENODEV;
8259 scsi_device_put(sdp);
8260 }
8261 } else {
8262 ret = -ENODEV;
8263 }
8264 spin_unlock_irqrestore(hba->host->host_lock, flags);
8265
8266 if (ret)
8267 return ret;
8268
8269
8270
8271
8272
8273
8274
8275 hba->host->eh_noresume = 1;
8276 if (hba->wlun_dev_clr_ua) {
8277 ret = ufshcd_send_request_sense(hba, sdp);
8278 if (ret)
8279 goto out;
8280
8281 hba->wlun_dev_clr_ua = false;
8282 }
8283
8284 cmd[4] = pwr_mode << 4;
8285
8286
8287
8288
8289
8290
8291 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8292 START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
8293 if (ret) {
8294 sdev_printk(KERN_WARNING, sdp,
8295 "START_STOP failed for power mode: %d, result %x\n",
8296 pwr_mode, ret);
8297 if (driver_byte(ret) == DRIVER_SENSE)
8298 scsi_print_sense_hdr(sdp, NULL, &sshdr);
8299 }
8300
8301 if (!ret)
8302 hba->curr_dev_pwr_mode = pwr_mode;
8303out:
8304 scsi_device_put(sdp);
8305 hba->host->eh_noresume = 0;
8306 return ret;
8307}
8308
8309static int ufshcd_link_state_transition(struct ufs_hba *hba,
8310 enum uic_link_state req_link_state,
8311 int check_for_bkops)
8312{
8313 int ret = 0;
8314
8315 if (req_link_state == hba->uic_link_state)
8316 return 0;
8317
8318 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8319 ret = ufshcd_uic_hibern8_enter(hba);
8320 if (!ret) {
8321 ufshcd_set_link_hibern8(hba);
8322 } else {
8323 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8324 __func__, ret);
8325 goto out;
8326 }
8327 }
8328
8329
8330
8331
8332 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
8333 (!check_for_bkops || !hba->auto_bkops_enabled)) {
8334
8335
8336
8337
8338
8339
8340
8341 ret = ufshcd_uic_hibern8_enter(hba);
8342 if (ret) {
8343 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8344 __func__, ret);
8345 goto out;
8346 }
8347
8348
8349
8350
8351 ufshcd_hba_stop(hba);
8352
8353
8354
8355
8356 ufshcd_set_link_off(hba);
8357 }
8358
8359out:
8360 return ret;
8361}
8362
8363static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8364{
8365 bool vcc_off = false;
8366
8367
8368
8369
8370
8371
8372
8373 if (!ufshcd_is_link_active(hba) &&
8374 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8375 usleep_range(2000, 2100);
8376
8377
8378
8379
8380
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8393 !hba->dev_info.is_lu_power_on_wp) {
8394 ufshcd_setup_vreg(hba, false);
8395 vcc_off = true;
8396 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8397 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8398 vcc_off = true;
8399 if (!ufshcd_is_link_active(hba)) {
8400 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8401 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8402 }
8403 }
8404
8405
8406
8407
8408 if (vcc_off && hba->vreg_info.vcc &&
8409 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8410 usleep_range(5000, 5100);
8411}
8412
8413static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8414{
8415 int ret = 0;
8416
8417 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8418 !hba->dev_info.is_lu_power_on_wp) {
8419 ret = ufshcd_setup_vreg(hba, true);
8420 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8421 if (!ret && !ufshcd_is_link_active(hba)) {
8422 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8423 if (ret)
8424 goto vcc_disable;
8425 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8426 if (ret)
8427 goto vccq_lpm;
8428 }
8429 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
8430 }
8431 goto out;
8432
8433vccq_lpm:
8434 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8435vcc_disable:
8436 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8437out:
8438 return ret;
8439}
8440
8441static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8442{
8443 if (ufshcd_is_link_off(hba))
8444 ufshcd_setup_hba_vreg(hba, false);
8445}
8446
8447static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8448{
8449 if (ufshcd_is_link_off(hba))
8450 ufshcd_setup_hba_vreg(hba, true);
8451}
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8470{
8471 int ret = 0;
8472 enum ufs_pm_level pm_lvl;
8473 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8474 enum uic_link_state req_link_state;
8475
8476 hba->pm_op_in_progress = 1;
8477 if (!ufshcd_is_shutdown_pm(pm_op)) {
8478 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
8479 hba->rpm_lvl : hba->spm_lvl;
8480 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8481 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8482 } else {
8483 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8484 req_link_state = UIC_LINK_OFF_STATE;
8485 }
8486
8487
8488
8489
8490
8491 ufshcd_hold(hba, false);
8492 hba->clk_gating.is_suspended = true;
8493
8494 if (hba->clk_scaling.is_allowed) {
8495 cancel_work_sync(&hba->clk_scaling.suspend_work);
8496 cancel_work_sync(&hba->clk_scaling.resume_work);
8497 ufshcd_suspend_clkscaling(hba);
8498 }
8499
8500 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8501 req_link_state == UIC_LINK_ACTIVE_STATE) {
8502 goto disable_clks;
8503 }
8504
8505 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8506 (req_link_state == hba->uic_link_state))
8507 goto enable_gating;
8508
8509
8510 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8511 ret = -EINVAL;
8512 goto enable_gating;
8513 }
8514
8515 if (ufshcd_is_runtime_pm(pm_op)) {
8516 if (ufshcd_can_autobkops_during_suspend(hba)) {
8517
8518
8519
8520
8521
8522 ret = ufshcd_urgent_bkops(hba);
8523 if (ret)
8524 goto enable_gating;
8525 } else {
8526
8527 ufshcd_disable_auto_bkops(hba);
8528 }
8529
8530
8531
8532
8533
8534 hba->dev_info.b_rpm_dev_flush_capable =
8535 hba->auto_bkops_enabled ||
8536 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8537 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8538 ufshcd_is_auto_hibern8_enabled(hba))) &&
8539 ufshcd_wb_need_flush(hba));
8540 }
8541
8542 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
8543 if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
8544 !ufshcd_is_runtime_pm(pm_op)) {
8545
8546 ufshcd_disable_auto_bkops(hba);
8547 }
8548
8549 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8550 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8551 if (ret)
8552 goto enable_gating;
8553 }
8554 }
8555
8556 flush_work(&hba->eeh_work);
8557 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
8558 if (ret)
8559 goto set_dev_active;
8560
8561 ufshcd_vreg_set_lpm(hba);
8562
8563disable_clks:
8564
8565
8566
8567
8568
8569 ret = ufshcd_vops_suspend(hba, pm_op);
8570 if (ret)
8571 goto set_link_active;
8572
8573
8574
8575
8576 ufshcd_disable_irq(hba);
8577
8578 if (!ufshcd_is_link_active(hba))
8579 ufshcd_setup_clocks(hba, false);
8580 else
8581
8582 __ufshcd_setup_clocks(hba, false, true);
8583
8584 if (ufshcd_is_clkgating_allowed(hba)) {
8585 hba->clk_gating.state = CLKS_OFF;
8586 trace_ufshcd_clk_gating(dev_name(hba->dev),
8587 hba->clk_gating.state);
8588 }
8589
8590
8591 ufshcd_hba_vreg_set_lpm(hba);
8592 goto out;
8593
8594set_link_active:
8595 if (hba->clk_scaling.is_allowed)
8596 ufshcd_resume_clkscaling(hba);
8597 ufshcd_vreg_set_hpm(hba);
8598 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8599 ufshcd_set_link_active(hba);
8600 else if (ufshcd_is_link_off(hba))
8601 ufshcd_host_reset_and_restore(hba);
8602set_dev_active:
8603 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8604 ufshcd_disable_auto_bkops(hba);
8605enable_gating:
8606 if (hba->clk_scaling.is_allowed)
8607 ufshcd_resume_clkscaling(hba);
8608 hba->clk_gating.is_suspended = false;
8609 hba->dev_info.b_rpm_dev_flush_capable = false;
8610 ufshcd_release(hba);
8611out:
8612 if (hba->dev_info.b_rpm_dev_flush_capable) {
8613 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8614 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8615 }
8616
8617 hba->pm_op_in_progress = 0;
8618
8619 if (ret)
8620 ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
8621 return ret;
8622}
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
8634static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8635{
8636 int ret;
8637 enum uic_link_state old_link_state;
8638
8639 hba->pm_op_in_progress = 1;
8640 old_link_state = hba->uic_link_state;
8641
8642 ufshcd_hba_vreg_set_hpm(hba);
8643
8644 ret = ufshcd_setup_clocks(hba, true);
8645 if (ret)
8646 goto out;
8647
8648
8649 ufshcd_enable_irq(hba);
8650
8651 ret = ufshcd_vreg_set_hpm(hba);
8652 if (ret)
8653 goto disable_irq_and_vops_clks;
8654
8655
8656
8657
8658
8659
8660 ret = ufshcd_vops_resume(hba, pm_op);
8661 if (ret)
8662 goto disable_vreg;
8663
8664 if (ufshcd_is_link_hibern8(hba)) {
8665 ret = ufshcd_uic_hibern8_exit(hba);
8666 if (!ret) {
8667 ufshcd_set_link_active(hba);
8668 } else {
8669 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8670 __func__, ret);
8671 goto vendor_suspend;
8672 }
8673 } else if (ufshcd_is_link_off(hba)) {
8674
8675
8676
8677
8678 ret = ufshcd_reset_and_restore(hba);
8679
8680
8681
8682
8683 if (ret || !ufshcd_is_link_active(hba))
8684 goto vendor_suspend;
8685 }
8686
8687 if (!ufshcd_is_ufs_dev_active(hba)) {
8688 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8689 if (ret)
8690 goto set_old_link_state;
8691 }
8692
8693 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8694 ufshcd_enable_auto_bkops(hba);
8695 else
8696
8697
8698
8699
8700 ufshcd_urgent_bkops(hba);
8701
8702 hba->clk_gating.is_suspended = false;
8703
8704 if (hba->clk_scaling.is_allowed)
8705 ufshcd_resume_clkscaling(hba);
8706
8707
8708 ufshcd_auto_hibern8_enable(hba);
8709
8710 if (hba->dev_info.b_rpm_dev_flush_capable) {
8711 hba->dev_info.b_rpm_dev_flush_capable = false;
8712 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8713 }
8714
8715
8716 ufshcd_release(hba);
8717
8718 goto out;
8719
8720set_old_link_state:
8721 ufshcd_link_state_transition(hba, old_link_state, 0);
8722vendor_suspend:
8723 ufshcd_vops_suspend(hba, pm_op);
8724disable_vreg:
8725 ufshcd_vreg_set_lpm(hba);
8726disable_irq_and_vops_clks:
8727 ufshcd_disable_irq(hba);
8728 if (hba->clk_scaling.is_allowed)
8729 ufshcd_suspend_clkscaling(hba);
8730 ufshcd_setup_clocks(hba, false);
8731 if (ufshcd_is_clkgating_allowed(hba)) {
8732 hba->clk_gating.state = CLKS_OFF;
8733 trace_ufshcd_clk_gating(dev_name(hba->dev),
8734 hba->clk_gating.state);
8735 }
8736out:
8737 hba->pm_op_in_progress = 0;
8738 if (ret)
8739 ufshcd_update_reg_hist(&hba->ufs_stats.resume_err, (u32)ret);
8740 return ret;
8741}
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751int ufshcd_system_suspend(struct ufs_hba *hba)
8752{
8753 int ret = 0;
8754 ktime_t start = ktime_get();
8755
8756 if (!hba || !hba->is_powered)
8757 return 0;
8758
8759 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
8760 hba->curr_dev_pwr_mode) &&
8761 (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
8762 hba->uic_link_state))
8763 goto out;
8764
8765 if (pm_runtime_suspended(hba->dev)) {
8766
8767
8768
8769
8770
8771
8772
8773
8774 ret = ufshcd_runtime_resume(hba);
8775 if (ret)
8776 goto out;
8777 }
8778
8779 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
8780out:
8781 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
8782 ktime_to_us(ktime_sub(ktime_get(), start)),
8783 hba->curr_dev_pwr_mode, hba->uic_link_state);
8784 if (!ret)
8785 hba->is_sys_suspended = true;
8786 return ret;
8787}
8788EXPORT_SYMBOL(ufshcd_system_suspend);
8789
8790
8791
8792
8793
8794
8795
8796
8797int ufshcd_system_resume(struct ufs_hba *hba)
8798{
8799 int ret = 0;
8800 ktime_t start = ktime_get();
8801
8802 if (!hba)
8803 return -EINVAL;
8804
8805 if (!hba->is_powered || pm_runtime_suspended(hba->dev))
8806
8807
8808
8809
8810 goto out;
8811 else
8812 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
8813out:
8814 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
8815 ktime_to_us(ktime_sub(ktime_get(), start)),
8816 hba->curr_dev_pwr_mode, hba->uic_link_state);
8817 if (!ret)
8818 hba->is_sys_suspended = false;
8819 return ret;
8820}
8821EXPORT_SYMBOL(ufshcd_system_resume);
8822
8823
8824
8825
8826
8827
8828
8829
8830
8831int ufshcd_runtime_suspend(struct ufs_hba *hba)
8832{
8833 int ret = 0;
8834 ktime_t start = ktime_get();
8835
8836 if (!hba)
8837 return -EINVAL;
8838
8839 if (!hba->is_powered)
8840 goto out;
8841 else
8842 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
8843out:
8844 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
8845 ktime_to_us(ktime_sub(ktime_get(), start)),
8846 hba->curr_dev_pwr_mode, hba->uic_link_state);
8847 return ret;
8848}
8849EXPORT_SYMBOL(ufshcd_runtime_suspend);
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872int ufshcd_runtime_resume(struct ufs_hba *hba)
8873{
8874 int ret = 0;
8875 ktime_t start = ktime_get();
8876
8877 if (!hba)
8878 return -EINVAL;
8879
8880 if (!hba->is_powered)
8881 goto out;
8882 else
8883 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
8884out:
8885 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
8886 ktime_to_us(ktime_sub(ktime_get(), start)),
8887 hba->curr_dev_pwr_mode, hba->uic_link_state);
8888 return ret;
8889}
8890EXPORT_SYMBOL(ufshcd_runtime_resume);
8891
8892int ufshcd_runtime_idle(struct ufs_hba *hba)
8893{
8894 return 0;
8895}
8896EXPORT_SYMBOL(ufshcd_runtime_idle);
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906int ufshcd_shutdown(struct ufs_hba *hba)
8907{
8908 int ret = 0;
8909
8910 if (!hba->is_powered)
8911 goto out;
8912
8913 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
8914 goto out;
8915
8916 pm_runtime_get_sync(hba->dev);
8917
8918 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
8919out:
8920 if (ret)
8921 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
8922
8923 return 0;
8924}
8925EXPORT_SYMBOL(ufshcd_shutdown);
8926
8927
8928
8929
8930
8931
8932void ufshcd_remove(struct ufs_hba *hba)
8933{
8934 ufs_bsg_remove(hba);
8935 ufs_sysfs_remove_nodes(hba->dev);
8936 blk_cleanup_queue(hba->tmf_queue);
8937 blk_mq_free_tag_set(&hba->tmf_tag_set);
8938 blk_cleanup_queue(hba->cmd_queue);
8939 scsi_remove_host(hba->host);
8940 destroy_workqueue(hba->eh_wq);
8941
8942 ufshcd_disable_intr(hba, hba->intr_mask);
8943 ufshcd_hba_stop(hba);
8944
8945 ufshcd_exit_clk_scaling(hba);
8946 ufshcd_exit_clk_gating(hba);
8947 if (ufshcd_is_clkscaling_supported(hba))
8948 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
8949 ufshcd_hba_exit(hba);
8950}
8951EXPORT_SYMBOL_GPL(ufshcd_remove);
8952
8953
8954
8955
8956
8957void ufshcd_dealloc_host(struct ufs_hba *hba)
8958{
8959 ufshcd_crypto_destroy_keyslot_manager(hba);
8960 scsi_host_put(hba->host);
8961}
8962EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
8963
8964
8965
8966
8967
8968
8969
8970
8971static int ufshcd_set_dma_mask(struct ufs_hba *hba)
8972{
8973 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
8974 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
8975 return 0;
8976 }
8977 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
8978}
8979
8980
8981
8982
8983
8984
8985
8986int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
8987{
8988 struct Scsi_Host *host;
8989 struct ufs_hba *hba;
8990 int err = 0;
8991
8992 if (!dev) {
8993 dev_err(dev,
8994 "Invalid memory reference for dev is NULL\n");
8995 err = -ENODEV;
8996 goto out_error;
8997 }
8998
8999 host = scsi_host_alloc(&ufshcd_driver_template,
9000 sizeof(struct ufs_hba));
9001 if (!host) {
9002 dev_err(dev, "scsi_host_alloc failed\n");
9003 err = -ENOMEM;
9004 goto out_error;
9005 }
9006 hba = shost_priv(host);
9007 hba->host = host;
9008 hba->dev = dev;
9009 *hba_handle = hba;
9010 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
9011
9012 INIT_LIST_HEAD(&hba->clk_list_head);
9013
9014out_error:
9015 return err;
9016}
9017EXPORT_SYMBOL(ufshcd_alloc_host);
9018
9019
9020static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9021 const struct blk_mq_queue_data *qd)
9022{
9023 WARN_ON_ONCE(true);
9024 return BLK_STS_NOTSUPP;
9025}
9026
9027static const struct blk_mq_ops ufshcd_tmf_ops = {
9028 .queue_rq = ufshcd_queue_tmf,
9029};
9030
9031
9032
9033
9034
9035
9036
9037
9038int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9039{
9040 int err;
9041 struct Scsi_Host *host = hba->host;
9042 struct device *dev = hba->dev;
9043 char eh_wq_name[sizeof("ufs_eh_wq_00")];
9044
9045 if (!mmio_base) {
9046 dev_err(hba->dev,
9047 "Invalid memory reference for mmio_base is NULL\n");
9048 err = -ENODEV;
9049 goto out_error;
9050 }
9051
9052 hba->mmio_base = mmio_base;
9053 hba->irq = irq;
9054 hba->vps = &ufs_hba_vps;
9055
9056 err = ufshcd_hba_init(hba);
9057 if (err)
9058 goto out_error;
9059
9060
9061 err = ufshcd_hba_capabilities(hba);
9062 if (err)
9063 goto out_disable;
9064
9065
9066 hba->ufs_version = ufshcd_get_ufs_version(hba);
9067
9068 if ((hba->ufs_version != UFSHCI_VERSION_10) &&
9069 (hba->ufs_version != UFSHCI_VERSION_11) &&
9070 (hba->ufs_version != UFSHCI_VERSION_20) &&
9071 (hba->ufs_version != UFSHCI_VERSION_21))
9072 dev_err(hba->dev, "invalid UFS version 0x%x\n",
9073 hba->ufs_version);
9074
9075
9076 hba->intr_mask = ufshcd_get_intr_mask(hba);
9077
9078 err = ufshcd_set_dma_mask(hba);
9079 if (err) {
9080 dev_err(hba->dev, "set dma mask failed\n");
9081 goto out_disable;
9082 }
9083
9084
9085 err = ufshcd_memory_alloc(hba);
9086 if (err) {
9087 dev_err(hba->dev, "Memory allocation failed\n");
9088 goto out_disable;
9089 }
9090
9091
9092 ufshcd_host_memory_configure(hba);
9093
9094 host->can_queue = hba->nutrs;
9095 host->cmd_per_lun = hba->nutrs;
9096 host->max_id = UFSHCD_MAX_ID;
9097 host->max_lun = UFS_MAX_LUNS;
9098 host->max_channel = UFSHCD_MAX_CHANNEL;
9099 host->unique_id = host->host_no;
9100 host->max_cmd_len = UFS_CDB_SIZE;
9101
9102 hba->max_pwr_info.is_valid = false;
9103
9104
9105 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9106 hba->host->host_no);
9107 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9108 if (!hba->eh_wq) {
9109 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9110 __func__);
9111 err = -ENOMEM;
9112 goto out_disable;
9113 }
9114 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
9115 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
9116
9117
9118 mutex_init(&hba->uic_cmd_mutex);
9119
9120
9121 mutex_init(&hba->dev_cmd.lock);
9122
9123 init_rwsem(&hba->clk_scaling_lock);
9124
9125 ufshcd_init_clk_gating(hba);
9126
9127 ufshcd_init_clk_scaling(hba);
9128
9129
9130
9131
9132
9133
9134 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9135 REG_INTERRUPT_STATUS);
9136 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9137
9138
9139
9140
9141 mb();
9142
9143
9144 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
9145 if (err) {
9146 dev_err(hba->dev, "request irq failed\n");
9147 goto exit_gating;
9148 } else {
9149 hba->is_irq_enabled = true;
9150 }
9151
9152 err = scsi_add_host(host, hba->dev);
9153 if (err) {
9154 dev_err(hba->dev, "scsi_add_host failed\n");
9155 goto exit_gating;
9156 }
9157
9158 hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9159 if (IS_ERR(hba->cmd_queue)) {
9160 err = PTR_ERR(hba->cmd_queue);
9161 goto out_remove_scsi_host;
9162 }
9163
9164 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9165 .nr_hw_queues = 1,
9166 .queue_depth = hba->nutmrs,
9167 .ops = &ufshcd_tmf_ops,
9168 .flags = BLK_MQ_F_NO_SCHED,
9169 };
9170 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9171 if (err < 0)
9172 goto free_cmd_queue;
9173 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9174 if (IS_ERR(hba->tmf_queue)) {
9175 err = PTR_ERR(hba->tmf_queue);
9176 goto free_tmf_tag_set;
9177 }
9178
9179
9180 ufshcd_vops_device_reset(hba);
9181
9182 ufshcd_init_crypto(hba);
9183
9184
9185 err = ufshcd_hba_enable(hba);
9186 if (err) {
9187 dev_err(hba->dev, "Host controller enable failed\n");
9188 ufshcd_print_host_regs(hba);
9189 ufshcd_print_host_state(hba);
9190 goto free_tmf_queue;
9191 }
9192
9193
9194
9195
9196
9197
9198 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9199 UFS_SLEEP_PWR_MODE,
9200 UIC_LINK_HIBERN8_STATE);
9201 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9202 UFS_SLEEP_PWR_MODE,
9203 UIC_LINK_HIBERN8_STATE);
9204
9205 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9206 ufshcd_rpm_dev_flush_recheck_work);
9207
9208
9209 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
9210 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9211 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9212 }
9213
9214
9215 pm_runtime_get_sync(dev);
9216 atomic_set(&hba->scsi_block_reqs_cnt, 0);
9217
9218
9219
9220
9221
9222
9223 ufshcd_set_ufs_dev_active(hba);
9224
9225 async_schedule(ufshcd_async_scan, hba);
9226 ufs_sysfs_add_nodes(hba->dev);
9227
9228 return 0;
9229
9230free_tmf_queue:
9231 blk_cleanup_queue(hba->tmf_queue);
9232free_tmf_tag_set:
9233 blk_mq_free_tag_set(&hba->tmf_tag_set);
9234free_cmd_queue:
9235 blk_cleanup_queue(hba->cmd_queue);
9236out_remove_scsi_host:
9237 scsi_remove_host(hba->host);
9238exit_gating:
9239 ufshcd_exit_clk_scaling(hba);
9240 ufshcd_exit_clk_gating(hba);
9241 destroy_workqueue(hba->eh_wq);
9242out_disable:
9243 hba->is_irq_enabled = false;
9244 ufshcd_hba_exit(hba);
9245out_error:
9246 return err;
9247}
9248EXPORT_SYMBOL_GPL(ufshcd_init);
9249
9250MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9251MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
9252MODULE_DESCRIPTION("Generic UFS host controller driver Core");
9253MODULE_LICENSE("GPL");
9254MODULE_VERSION(UFSHCD_DRIVER_VERSION);
9255
