1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#include <linux/async.h>
41#include <linux/devfreq.h>
42#include <linux/nls.h>
43#include <linux/of.h>
44#include "ufshcd.h"
45#include "ufs_quirks.h"
46#include "unipro.h"
47
48#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
49 UTP_TASK_REQ_COMPL |\
50 UFSHCD_ERROR_MASK)
51
52#define UIC_CMD_TIMEOUT 500
53
54
55#define NOP_OUT_RETRIES 10
56
57#define NOP_OUT_TIMEOUT 30
58
59
60#define QUERY_REQ_RETRIES 10
61
62#define QUERY_REQ_TIMEOUT 30
63
64
65
66
67
68#define QUERY_FDEVICEINIT_REQ_TIMEOUT 600
69
70
71#define TM_CMD_TIMEOUT 100
72
73
74#define UFS_UIC_COMMAND_RETRIES 3
75
76
77#define DME_LINKSTARTUP_RETRIES 3
78
79
80#define UIC_HIBERN8_ENTER_RETRIES 3
81
82
83#define MAX_HOST_RESET_RETRIES 5
84
85
86#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87
88
89#define INT_AGGR_DEF_TO 0x02
90
91#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
92 ({ \
93 int _ret; \
94 if (_on) \
95 _ret = ufshcd_enable_vreg(_dev, _vreg); \
96 else \
97 _ret = ufshcd_disable_vreg(_dev, _vreg); \
98 _ret; \
99 })
100
101static u32 ufs_query_desc_max_size[] = {
102 QUERY_DESC_DEVICE_MAX_SIZE,
103 QUERY_DESC_CONFIGURAION_MAX_SIZE,
104 QUERY_DESC_UNIT_MAX_SIZE,
105 QUERY_DESC_RFU_MAX_SIZE,
106 QUERY_DESC_INTERCONNECT_MAX_SIZE,
107 QUERY_DESC_STRING_MAX_SIZE,
108 QUERY_DESC_RFU_MAX_SIZE,
109 QUERY_DESC_GEOMETRY_MAX_SIZE,
110 QUERY_DESC_POWER_MAX_SIZE,
111 QUERY_DESC_RFU_MAX_SIZE,
112};
113
114enum {
115 UFSHCD_MAX_CHANNEL = 0,
116 UFSHCD_MAX_ID = 1,
117 UFSHCD_CMD_PER_LUN = 32,
118 UFSHCD_CAN_QUEUE = 32,
119};
120
121
122enum {
123 UFSHCD_STATE_RESET,
124 UFSHCD_STATE_ERROR,
125 UFSHCD_STATE_OPERATIONAL,
126};
127
128
129enum {
130 UFSHCD_EH_IN_PROGRESS = (1 << 0),
131};
132
133
134enum {
135 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0),
136 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1),
137 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2),
138 UFSHCD_UIC_NL_ERROR = (1 << 3),
139 UFSHCD_UIC_TL_ERROR = (1 << 4),
140 UFSHCD_UIC_DME_ERROR = (1 << 5),
141};
142
143
144enum {
145 UFSHCD_INT_DISABLE,
146 UFSHCD_INT_ENABLE,
147 UFSHCD_INT_CLEAR,
148};
149
150#define ufshcd_set_eh_in_progress(h) \
151 (h->eh_flags |= UFSHCD_EH_IN_PROGRESS)
152#define ufshcd_eh_in_progress(h) \
153 (h->eh_flags & UFSHCD_EH_IN_PROGRESS)
154#define ufshcd_clear_eh_in_progress(h) \
155 (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
156
157#define ufshcd_set_ufs_dev_active(h) \
158 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
159#define ufshcd_set_ufs_dev_sleep(h) \
160 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
161#define ufshcd_set_ufs_dev_poweroff(h) \
162 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
163#define ufshcd_is_ufs_dev_active(h) \
164 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
165#define ufshcd_is_ufs_dev_sleep(h) \
166 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
167#define ufshcd_is_ufs_dev_poweroff(h) \
168 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
169
170static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
171 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
172 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
173 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
174 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
175 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
176 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
177};
178
179static inline enum ufs_dev_pwr_mode
180ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
181{
182 return ufs_pm_lvl_states[lvl].dev_state;
183}
184
185static inline enum uic_link_state
186ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
187{
188 return ufs_pm_lvl_states[lvl].link_state;
189}
190
191static void ufshcd_tmc_handler(struct ufs_hba *hba);
192static void ufshcd_async_scan(void *data, async_cookie_t cookie);
193static int ufshcd_reset_and_restore(struct ufs_hba *hba);
194static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
195static void ufshcd_hba_exit(struct ufs_hba *hba);
196static int ufshcd_probe_hba(struct ufs_hba *hba);
197static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
198 bool skip_ref_clk);
199static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
200static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
201static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
202static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
203static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
204static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
205static irqreturn_t ufshcd_intr(int irq, void *__hba);
206static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
207 struct ufs_pa_layer_attr *desired_pwr_mode);
208static int ufshcd_change_power_mode(struct ufs_hba *hba,
209 struct ufs_pa_layer_attr *pwr_mode);
210static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
211{
212 return tag >= 0 && tag < hba->nutrs;
213}
214
215static inline int ufshcd_enable_irq(struct ufs_hba *hba)
216{
217 int ret = 0;
218
219 if (!hba->is_irq_enabled) {
220 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
221 hba);
222 if (ret)
223 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
224 __func__, ret);
225 hba->is_irq_enabled = true;
226 }
227
228 return ret;
229}
230
231static inline void ufshcd_disable_irq(struct ufs_hba *hba)
232{
233 if (hba->is_irq_enabled) {
234 free_irq(hba->irq, hba);
235 hba->is_irq_enabled = false;
236 }
237}
238
239
240static inline void ufshcd_remove_non_printable(char *val)
241{
242 if (!val)
243 return;
244
245 if (*val < 0x20 || *val > 0x7e)
246 *val = ' ';
247}
248
249
250
251
252
253
254
255
256
257
258
259
260
261int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
262 u32 val, unsigned long interval_us,
263 unsigned long timeout_ms, bool can_sleep)
264{
265 int err = 0;
266 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
267
268
269 val = val & mask;
270
271 while ((ufshcd_readl(hba, reg) & mask) != val) {
272 if (can_sleep)
273 usleep_range(interval_us, interval_us + 50);
274 else
275 udelay(interval_us);
276 if (time_after(jiffies, timeout)) {
277 if ((ufshcd_readl(hba, reg) & mask) != val)
278 err = -ETIMEDOUT;
279 break;
280 }
281 }
282
283 return err;
284}
285
286
287
288
289
290
291
292static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
293{
294 if (hba->ufs_version == UFSHCI_VERSION_10)
295 return INTERRUPT_MASK_ALL_VER_10;
296 else
297 return INTERRUPT_MASK_ALL_VER_11;
298}
299
300
301
302
303
304
305
306static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
307{
308 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
309 return ufshcd_vops_get_ufs_hci_version(hba);
310
311 return ufshcd_readl(hba, REG_UFS_VERSION);
312}
313
314
315
316
317
318
319
320
321static inline int ufshcd_is_device_present(struct ufs_hba *hba)
322{
323 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
324 DEVICE_PRESENT) ? 1 : 0;
325}
326
327
328
329
330
331
332
333
334static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
335{
336 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
337}
338
339
340
341
342
343
344
345
346static inline int
347ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
348{
349 return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
350}
351
352
353
354
355
356
357
358
359
360
361static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
362{
363 int tag;
364 bool ret = false;
365
366 if (!free_slot)
367 goto out;
368
369 do {
370 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
371 if (tag >= hba->nutmrs)
372 goto out;
373 } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
374
375 *free_slot = tag;
376 ret = true;
377out:
378 return ret;
379}
380
381static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
382{
383 clear_bit_unlock(slot, &hba->tm_slots_in_use);
384}
385
386
387
388
389
390
391static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
392{
393 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
394}
395
396
397
398
399
400
401static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
402{
403 __clear_bit(tag, &hba->outstanding_reqs);
404}
405
406
407
408
409
410
411
412static inline int ufshcd_get_lists_status(u32 reg)
413{
414
415
416
417
418
419
420
421
422
423 return ((reg & 0xFF) >> 1) ^ 0x07;
424}
425
426
427
428
429
430
431
432
433static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
434{
435 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
436 MASK_UIC_COMMAND_RESULT;
437}
438
439
440
441
442
443
444
445
446static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
447{
448 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
449}
450
451
452
453
454
455static inline int
456ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
457{
458 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
459}
460
461
462
463
464
465
466
467
468static inline int
469ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
470{
471 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
472}
473
474
475
476
477
478
479
480
481static inline unsigned int
482ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
483{
484 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
485 MASK_RSP_UPIU_DATA_SEG_LEN;
486}
487
488
489
490
491
492
493
494
495
496
497static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
498{
499 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
500 MASK_RSP_EXCEPTION_EVENT ? true : false;
501}
502
503
504
505
506
507static inline void
508ufshcd_reset_intr_aggr(struct ufs_hba *hba)
509{
510 ufshcd_writel(hba, INT_AGGR_ENABLE |
511 INT_AGGR_COUNTER_AND_TIMER_RESET,
512 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
513}
514
515
516
517
518
519
520
521static inline void
522ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
523{
524 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
525 INT_AGGR_COUNTER_THLD_VAL(cnt) |
526 INT_AGGR_TIMEOUT_VAL(tmout),
527 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
528}
529
530
531
532
533
534static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
535{
536 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
537}
538
539
540
541
542
543
544
545static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
546{
547 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
548 REG_UTP_TASK_REQ_LIST_RUN_STOP);
549 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
550 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
551}
552
553
554
555
556
557static inline void ufshcd_hba_start(struct ufs_hba *hba)
558{
559 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
560}
561
562
563
564
565
566
567
568static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
569{
570 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
571}
572
573u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
574{
575
576 if ((hba->ufs_version == UFSHCI_VERSION_10) ||
577 (hba->ufs_version == UFSHCI_VERSION_11))
578 return UFS_UNIPRO_VER_1_41;
579 else
580 return UFS_UNIPRO_VER_1_6;
581}
582EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
583
584static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
585{
586
587
588
589
590
591
592
593
594
595 if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
596 return true;
597 else
598 return false;
599}
600
601static void ufshcd_ungate_work(struct work_struct *work)
602{
603 int ret;
604 unsigned long flags;
605 struct ufs_hba *hba = container_of(work, struct ufs_hba,
606 clk_gating.ungate_work);
607
608 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
609
610 spin_lock_irqsave(hba->host->host_lock, flags);
611 if (hba->clk_gating.state == CLKS_ON) {
612 spin_unlock_irqrestore(hba->host->host_lock, flags);
613 goto unblock_reqs;
614 }
615
616 spin_unlock_irqrestore(hba->host->host_lock, flags);
617 ufshcd_setup_clocks(hba, true);
618
619
620 if (ufshcd_can_hibern8_during_gating(hba)) {
621
622 hba->clk_gating.is_suspended = true;
623 if (ufshcd_is_link_hibern8(hba)) {
624 ret = ufshcd_uic_hibern8_exit(hba);
625 if (ret)
626 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
627 __func__, ret);
628 else
629 ufshcd_set_link_active(hba);
630 }
631 hba->clk_gating.is_suspended = false;
632 }
633unblock_reqs:
634 if (ufshcd_is_clkscaling_enabled(hba))
635 devfreq_resume_device(hba->devfreq);
636 scsi_unblock_requests(hba->host);
637}
638
639
640
641
642
643
644
645int ufshcd_hold(struct ufs_hba *hba, bool async)
646{
647 int rc = 0;
648 unsigned long flags;
649
650 if (!ufshcd_is_clkgating_allowed(hba))
651 goto out;
652 spin_lock_irqsave(hba->host->host_lock, flags);
653 hba->clk_gating.active_reqs++;
654
655 if (ufshcd_eh_in_progress(hba)) {
656 spin_unlock_irqrestore(hba->host->host_lock, flags);
657 return 0;
658 }
659
660start:
661 switch (hba->clk_gating.state) {
662 case CLKS_ON:
663 break;
664 case REQ_CLKS_OFF:
665 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
666 hba->clk_gating.state = CLKS_ON;
667 break;
668 }
669
670
671
672
673
674 case CLKS_OFF:
675 scsi_block_requests(hba->host);
676 hba->clk_gating.state = REQ_CLKS_ON;
677 schedule_work(&hba->clk_gating.ungate_work);
678
679
680
681
682 case REQ_CLKS_ON:
683 if (async) {
684 rc = -EAGAIN;
685 hba->clk_gating.active_reqs--;
686 break;
687 }
688
689 spin_unlock_irqrestore(hba->host->host_lock, flags);
690 flush_work(&hba->clk_gating.ungate_work);
691
692 spin_lock_irqsave(hba->host->host_lock, flags);
693 goto start;
694 default:
695 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
696 __func__, hba->clk_gating.state);
697 break;
698 }
699 spin_unlock_irqrestore(hba->host->host_lock, flags);
700out:
701 return rc;
702}
703EXPORT_SYMBOL_GPL(ufshcd_hold);
704
705static void ufshcd_gate_work(struct work_struct *work)
706{
707 struct ufs_hba *hba = container_of(work, struct ufs_hba,
708 clk_gating.gate_work.work);
709 unsigned long flags;
710
711 spin_lock_irqsave(hba->host->host_lock, flags);
712 if (hba->clk_gating.is_suspended) {
713 hba->clk_gating.state = CLKS_ON;
714 goto rel_lock;
715 }
716
717 if (hba->clk_gating.active_reqs
718 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
719 || hba->lrb_in_use || hba->outstanding_tasks
720 || hba->active_uic_cmd || hba->uic_async_done)
721 goto rel_lock;
722
723 spin_unlock_irqrestore(hba->host->host_lock, flags);
724
725
726 if (ufshcd_can_hibern8_during_gating(hba)) {
727 if (ufshcd_uic_hibern8_enter(hba)) {
728 hba->clk_gating.state = CLKS_ON;
729 goto out;
730 }
731 ufshcd_set_link_hibern8(hba);
732 }
733
734 if (ufshcd_is_clkscaling_enabled(hba)) {
735 devfreq_suspend_device(hba->devfreq);
736 hba->clk_scaling.window_start_t = 0;
737 }
738
739 if (!ufshcd_is_link_active(hba))
740 ufshcd_setup_clocks(hba, false);
741 else
742
743 __ufshcd_setup_clocks(hba, false, true);
744
745
746
747
748
749
750
751
752
753
754 spin_lock_irqsave(hba->host->host_lock, flags);
755 if (hba->clk_gating.state == REQ_CLKS_OFF)
756 hba->clk_gating.state = CLKS_OFF;
757
758rel_lock:
759 spin_unlock_irqrestore(hba->host->host_lock, flags);
760out:
761 return;
762}
763
764
765static void __ufshcd_release(struct ufs_hba *hba)
766{
767 if (!ufshcd_is_clkgating_allowed(hba))
768 return;
769
770 hba->clk_gating.active_reqs--;
771
772 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
773 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
774 || hba->lrb_in_use || hba->outstanding_tasks
775 || hba->active_uic_cmd || hba->uic_async_done
776 || ufshcd_eh_in_progress(hba))
777 return;
778
779 hba->clk_gating.state = REQ_CLKS_OFF;
780 schedule_delayed_work(&hba->clk_gating.gate_work,
781 msecs_to_jiffies(hba->clk_gating.delay_ms));
782}
783
784void ufshcd_release(struct ufs_hba *hba)
785{
786 unsigned long flags;
787
788 spin_lock_irqsave(hba->host->host_lock, flags);
789 __ufshcd_release(hba);
790 spin_unlock_irqrestore(hba->host->host_lock, flags);
791}
792EXPORT_SYMBOL_GPL(ufshcd_release);
793
794static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
795 struct device_attribute *attr, char *buf)
796{
797 struct ufs_hba *hba = dev_get_drvdata(dev);
798
799 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
800}
801
802static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
803 struct device_attribute *attr, const char *buf, size_t count)
804{
805 struct ufs_hba *hba = dev_get_drvdata(dev);
806 unsigned long flags, value;
807
808 if (kstrtoul(buf, 0, &value))
809 return -EINVAL;
810
811 spin_lock_irqsave(hba->host->host_lock, flags);
812 hba->clk_gating.delay_ms = value;
813 spin_unlock_irqrestore(hba->host->host_lock, flags);
814 return count;
815}
816
817static void ufshcd_init_clk_gating(struct ufs_hba *hba)
818{
819 if (!ufshcd_is_clkgating_allowed(hba))
820 return;
821
822 hba->clk_gating.delay_ms = 150;
823 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
824 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
825
826 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
827 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
828 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
829 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
830 hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR;
831 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
832 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
833}
834
835static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
836{
837 if (!ufshcd_is_clkgating_allowed(hba))
838 return;
839 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
840 cancel_work_sync(&hba->clk_gating.ungate_work);
841 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
842}
843
844
845static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
846{
847 if (!ufshcd_is_clkscaling_enabled(hba))
848 return;
849
850 if (!hba->clk_scaling.is_busy_started) {
851 hba->clk_scaling.busy_start_t = ktime_get();
852 hba->clk_scaling.is_busy_started = true;
853 }
854}
855
856static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
857{
858 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
859
860 if (!ufshcd_is_clkscaling_enabled(hba))
861 return;
862
863 if (!hba->outstanding_reqs && scaling->is_busy_started) {
864 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
865 scaling->busy_start_t));
866 scaling->busy_start_t = ktime_set(0, 0);
867 scaling->is_busy_started = false;
868 }
869}
870
871
872
873
874
875static inline
876void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
877{
878 ufshcd_clk_scaling_start_busy(hba);
879 __set_bit(task_tag, &hba->outstanding_reqs);
880 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
881}
882
883
884
885
886
887static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
888{
889 int len;
890 if (lrbp->sense_buffer &&
891 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
892 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
893 memcpy(lrbp->sense_buffer,
894 lrbp->ucd_rsp_ptr->sr.sense_data,
895 min_t(int, len, SCSI_SENSE_BUFFERSIZE));
896 }
897}
898
899
900
901
902
903
904
905static
906int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
907{
908 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
909
910 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
911
912
913 if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
914 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
915 GENERAL_UPIU_REQUEST_SIZE;
916 u16 resp_len;
917 u16 buf_len;
918
919
920 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
921 MASK_QUERY_DATA_SEG_LEN;
922 buf_len = be16_to_cpu(
923 hba->dev_cmd.query.request.upiu_req.length);
924 if (likely(buf_len >= resp_len)) {
925 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
926 } else {
927 dev_warn(hba->dev,
928 "%s: Response size is bigger than buffer",
929 __func__);
930 return -EINVAL;
931 }
932 }
933
934 return 0;
935}
936
937
938
939
940
941static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
942{
943 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
944
945
946 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
947 hba->nutmrs =
948 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
949}
950
951
952
953
954
955
956
957static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
958{
959 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
960 return true;
961 else
962 return false;
963}
964
965
966
967
968
969
970
971
972static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
973{
974 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
975}
976
977
978
979
980
981
982
983
984static inline void
985ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
986{
987 WARN_ON(hba->active_uic_cmd);
988
989 hba->active_uic_cmd = uic_cmd;
990
991
992 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
993 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
994 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
995
996
997 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
998 REG_UIC_COMMAND);
999}
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009static int
1010ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1011{
1012 int ret;
1013 unsigned long flags;
1014
1015 if (wait_for_completion_timeout(&uic_cmd->done,
1016 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
1017 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
1018 else
1019 ret = -ETIMEDOUT;
1020
1021 spin_lock_irqsave(hba->host->host_lock, flags);
1022 hba->active_uic_cmd = NULL;
1023 spin_unlock_irqrestore(hba->host->host_lock, flags);
1024
1025 return ret;
1026}
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038static int
1039__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
1040 bool completion)
1041{
1042 if (!ufshcd_ready_for_uic_cmd(hba)) {
1043 dev_err(hba->dev,
1044 "Controller not ready to accept UIC commands\n");
1045 return -EIO;
1046 }
1047
1048 if (completion)
1049 init_completion(&uic_cmd->done);
1050
1051 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
1052
1053 return 0;
1054}
1055
1056
1057
1058
1059
1060
1061
1062
1063static int
1064ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1065{
1066 int ret;
1067 unsigned long flags;
1068
1069 ufshcd_hold(hba, false);
1070 mutex_lock(&hba->uic_cmd_mutex);
1071 ufshcd_add_delay_before_dme_cmd(hba);
1072
1073 spin_lock_irqsave(hba->host->host_lock, flags);
1074 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
1075 spin_unlock_irqrestore(hba->host->host_lock, flags);
1076 if (!ret)
1077 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
1078
1079 mutex_unlock(&hba->uic_cmd_mutex);
1080
1081 ufshcd_release(hba);
1082 return ret;
1083}
1084
1085
1086
1087
1088
1089
1090
1091static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
1092{
1093 struct ufshcd_sg_entry *prd_table;
1094 struct scatterlist *sg;
1095 struct scsi_cmnd *cmd;
1096 int sg_segments;
1097 int i;
1098
1099 cmd = lrbp->cmd;
1100 sg_segments = scsi_dma_map(cmd);
1101 if (sg_segments < 0)
1102 return sg_segments;
1103
1104 if (sg_segments) {
1105 lrbp->utr_descriptor_ptr->prd_table_length =
1106 cpu_to_le16((u16) (sg_segments));
1107
1108 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
1109
1110 scsi_for_each_sg(cmd, sg, sg_segments, i) {
1111 prd_table[i].size =
1112 cpu_to_le32(((u32) sg_dma_len(sg))-1);
1113 prd_table[i].base_addr =
1114 cpu_to_le32(lower_32_bits(sg->dma_address));
1115 prd_table[i].upper_addr =
1116 cpu_to_le32(upper_32_bits(sg->dma_address));
1117 prd_table[i].reserved = 0;
1118 }
1119 } else {
1120 lrbp->utr_descriptor_ptr->prd_table_length = 0;
1121 }
1122
1123 return 0;
1124}
1125
1126
1127
1128
1129
1130
1131static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
1132{
1133 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1134
1135 if (hba->ufs_version == UFSHCI_VERSION_10) {
1136 u32 rw;
1137 rw = set & INTERRUPT_MASK_RW_VER_10;
1138 set = rw | ((set ^ intrs) & intrs);
1139 } else {
1140 set |= intrs;
1141 }
1142
1143 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1144}
1145
1146
1147
1148
1149
1150
1151static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
1152{
1153 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1154
1155 if (hba->ufs_version == UFSHCI_VERSION_10) {
1156 u32 rw;
1157 rw = (set & INTERRUPT_MASK_RW_VER_10) &
1158 ~(intrs & INTERRUPT_MASK_RW_VER_10);
1159 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
1160
1161 } else {
1162 set &= ~intrs;
1163 }
1164
1165 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1166}
1167
1168
1169
1170
1171
1172
1173
1174
1175static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
1176 u32 *upiu_flags, enum dma_data_direction cmd_dir)
1177{
1178 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
1179 u32 data_direction;
1180 u32 dword_0;
1181
1182 if (cmd_dir == DMA_FROM_DEVICE) {
1183 data_direction = UTP_DEVICE_TO_HOST;
1184 *upiu_flags = UPIU_CMD_FLAGS_READ;
1185 } else if (cmd_dir == DMA_TO_DEVICE) {
1186 data_direction = UTP_HOST_TO_DEVICE;
1187 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
1188 } else {
1189 data_direction = UTP_NO_DATA_TRANSFER;
1190 *upiu_flags = UPIU_CMD_FLAGS_NONE;
1191 }
1192
1193 dword_0 = data_direction | (lrbp->command_type
1194 << UPIU_COMMAND_TYPE_OFFSET);
1195 if (lrbp->intr_cmd)
1196 dword_0 |= UTP_REQ_DESC_INT_CMD;
1197
1198
1199 req_desc->header.dword_0 = cpu_to_le32(dword_0);
1200
1201 req_desc->header.dword_1 = 0;
1202
1203
1204
1205
1206
1207 req_desc->header.dword_2 =
1208 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
1209
1210 req_desc->header.dword_3 = 0;
1211
1212 req_desc->prd_table_length = 0;
1213}
1214
1215
1216
1217
1218
1219
1220
1221static
1222void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
1223{
1224 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1225 unsigned short cdb_len;
1226
1227
1228 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1229 UPIU_TRANSACTION_COMMAND, upiu_flags,
1230 lrbp->lun, lrbp->task_tag);
1231 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1232 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
1233
1234
1235 ucd_req_ptr->header.dword_2 = 0;
1236
1237 ucd_req_ptr->sc.exp_data_transfer_len =
1238 cpu_to_be32(lrbp->cmd->sdb.length);
1239
1240 cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
1241 memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
1242 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
1243
1244 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
1255 struct ufshcd_lrb *lrbp, u32 upiu_flags)
1256{
1257 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1258 struct ufs_query *query = &hba->dev_cmd.query;
1259 u16 len = be16_to_cpu(query->request.upiu_req.length);
1260 u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
1261
1262
1263 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1264 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
1265 lrbp->lun, lrbp->task_tag);
1266 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1267 0, query->request.query_func, 0, 0);
1268
1269
1270 ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD(
1271 0, 0, len >> 8, (u8)len);
1272
1273
1274 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
1275 QUERY_OSF_SIZE);
1276
1277
1278 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
1279 memcpy(descp, query->descriptor, len);
1280
1281 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1282}
1283
1284static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
1285{
1286 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1287
1288 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
1289
1290
1291 ucd_req_ptr->header.dword_0 =
1292 UPIU_HEADER_DWORD(
1293 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
1294
1295 ucd_req_ptr->header.dword_1 = 0;
1296 ucd_req_ptr->header.dword_2 = 0;
1297
1298 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1299}
1300
1301
1302
1303
1304
1305
1306static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1307{
1308 u32 upiu_flags;
1309 int ret = 0;
1310
1311 switch (lrbp->command_type) {
1312 case UTP_CMD_TYPE_SCSI:
1313 if (likely(lrbp->cmd)) {
1314 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
1315 lrbp->cmd->sc_data_direction);
1316 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
1317 } else {
1318 ret = -EINVAL;
1319 }
1320 break;
1321 case UTP_CMD_TYPE_DEV_MANAGE:
1322 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
1323 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
1324 ufshcd_prepare_utp_query_req_upiu(
1325 hba, lrbp, upiu_flags);
1326 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
1327 ufshcd_prepare_utp_nop_upiu(lrbp);
1328 else
1329 ret = -EINVAL;
1330 break;
1331 case UTP_CMD_TYPE_UFS:
1332
1333 ret = -ENOTSUPP;
1334 dev_err(hba->dev, "%s: UFS native command are not supported\n",
1335 __func__);
1336 break;
1337 default:
1338 ret = -ENOTSUPP;
1339 dev_err(hba->dev, "%s: unknown command type: 0x%x\n",
1340 __func__, lrbp->command_type);
1341 break;
1342 }
1343
1344 return ret;
1345}
1346
1347
1348
1349
1350
1351
1352
1353static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
1354{
1355 if (scsi_is_wlun(scsi_lun))
1356 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
1357 | UFS_UPIU_WLUN_ID;
1358 else
1359 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
1360}
1361
1362
1363
1364
1365
1366
1367
1368static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
1369{
1370 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
1371}
1372
1373
1374
1375
1376
1377
1378
1379
1380static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
1381{
1382 struct ufshcd_lrb *lrbp;
1383 struct ufs_hba *hba;
1384 unsigned long flags;
1385 int tag;
1386 int err = 0;
1387
1388 hba = shost_priv(host);
1389
1390 tag = cmd->request->tag;
1391 if (!ufshcd_valid_tag(hba, tag)) {
1392 dev_err(hba->dev,
1393 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
1394 __func__, tag, cmd, cmd->request);
1395 BUG();
1396 }
1397
1398 spin_lock_irqsave(hba->host->host_lock, flags);
1399 switch (hba->ufshcd_state) {
1400 case UFSHCD_STATE_OPERATIONAL:
1401 break;
1402 case UFSHCD_STATE_RESET:
1403 err = SCSI_MLQUEUE_HOST_BUSY;
1404 goto out_unlock;
1405 case UFSHCD_STATE_ERROR:
1406 set_host_byte(cmd, DID_ERROR);
1407 cmd->scsi_done(cmd);
1408 goto out_unlock;
1409 default:
1410 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
1411 __func__, hba->ufshcd_state);
1412 set_host_byte(cmd, DID_BAD_TARGET);
1413 cmd->scsi_done(cmd);
1414 goto out_unlock;
1415 }
1416
1417
1418 if (ufshcd_eh_in_progress(hba)) {
1419 set_host_byte(cmd, DID_ERROR);
1420 cmd->scsi_done(cmd);
1421 goto out_unlock;
1422 }
1423 spin_unlock_irqrestore(hba->host->host_lock, flags);
1424
1425
1426 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
1427
1428
1429
1430
1431
1432
1433 err = SCSI_MLQUEUE_HOST_BUSY;
1434 goto out;
1435 }
1436
1437 err = ufshcd_hold(hba, true);
1438 if (err) {
1439 err = SCSI_MLQUEUE_HOST_BUSY;
1440 clear_bit_unlock(tag, &hba->lrb_in_use);
1441 goto out;
1442 }
1443 WARN_ON(hba->clk_gating.state != CLKS_ON);
1444
1445 lrbp = &hba->lrb[tag];
1446
1447 WARN_ON(lrbp->cmd);
1448 lrbp->cmd = cmd;
1449 lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
1450 lrbp->sense_buffer = cmd->sense_buffer;
1451 lrbp->task_tag = tag;
1452 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
1453 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
1454 lrbp->command_type = UTP_CMD_TYPE_SCSI;
1455
1456
1457 ufshcd_compose_upiu(hba, lrbp);
1458 err = ufshcd_map_sg(lrbp);
1459 if (err) {
1460 lrbp->cmd = NULL;
1461 clear_bit_unlock(tag, &hba->lrb_in_use);
1462 goto out;
1463 }
1464
1465
1466 spin_lock_irqsave(hba->host->host_lock, flags);
1467 ufshcd_send_command(hba, tag);
1468out_unlock:
1469 spin_unlock_irqrestore(hba->host->host_lock, flags);
1470out:
1471 return err;
1472}
1473
1474static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
1475 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
1476{
1477 lrbp->cmd = NULL;
1478 lrbp->sense_bufflen = 0;
1479 lrbp->sense_buffer = NULL;
1480 lrbp->task_tag = tag;
1481 lrbp->lun = 0;
1482 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
1483 lrbp->intr_cmd = true;
1484 hba->dev_cmd.type = cmd_type;
1485
1486 return ufshcd_compose_upiu(hba, lrbp);
1487}
1488
1489static int
1490ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
1491{
1492 int err = 0;
1493 unsigned long flags;
1494 u32 mask = 1 << tag;
1495
1496
1497 spin_lock_irqsave(hba->host->host_lock, flags);
1498 ufshcd_utrl_clear(hba, tag);
1499 spin_unlock_irqrestore(hba->host->host_lock, flags);
1500
1501
1502
1503
1504
1505 err = ufshcd_wait_for_register(hba,
1506 REG_UTP_TRANSFER_REQ_DOOR_BELL,
1507 mask, ~mask, 1000, 1000, true);
1508
1509 return err;
1510}
1511
1512static int
1513ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1514{
1515 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1516
1517
1518 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
1519 UPIU_RSP_CODE_OFFSET;
1520 return query_res->response;
1521}
1522
1523
1524
1525
1526
1527
1528static int
1529ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1530{
1531 int resp;
1532 int err = 0;
1533
1534 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
1535
1536 switch (resp) {
1537 case UPIU_TRANSACTION_NOP_IN:
1538 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
1539 err = -EINVAL;
1540 dev_err(hba->dev, "%s: unexpected response %x\n",
1541 __func__, resp);
1542 }
1543 break;
1544 case UPIU_TRANSACTION_QUERY_RSP:
1545 err = ufshcd_check_query_response(hba, lrbp);
1546 if (!err)
1547 err = ufshcd_copy_query_response(hba, lrbp);
1548 break;
1549 case UPIU_TRANSACTION_REJECT_UPIU:
1550
1551 err = -EPERM;
1552 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
1553 __func__);
1554 break;
1555 default:
1556 err = -EINVAL;
1557 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
1558 __func__, resp);
1559 break;
1560 }
1561
1562 return err;
1563}
1564
1565static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
1566 struct ufshcd_lrb *lrbp, int max_timeout)
1567{
1568 int err = 0;
1569 unsigned long time_left;
1570 unsigned long flags;
1571
1572 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
1573 msecs_to_jiffies(max_timeout));
1574
1575 spin_lock_irqsave(hba->host->host_lock, flags);
1576 hba->dev_cmd.complete = NULL;
1577 if (likely(time_left)) {
1578 err = ufshcd_get_tr_ocs(lrbp);
1579 if (!err)
1580 err = ufshcd_dev_cmd_completion(hba, lrbp);
1581 }
1582 spin_unlock_irqrestore(hba->host->host_lock, flags);
1583
1584 if (!time_left) {
1585 err = -ETIMEDOUT;
1586 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
1587 __func__, lrbp->task_tag);
1588 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
1589
1590 err = -EAGAIN;
1591
1592
1593
1594
1595
1596 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
1597 }
1598
1599 return err;
1600}
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
1614{
1615 int tag;
1616 bool ret = false;
1617 unsigned long tmp;
1618
1619 if (!tag_out)
1620 goto out;
1621
1622 do {
1623 tmp = ~hba->lrb_in_use;
1624 tag = find_last_bit(&tmp, hba->nutrs);
1625 if (tag >= hba->nutrs)
1626 goto out;
1627 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
1628
1629 *tag_out = tag;
1630 ret = true;
1631out:
1632 return ret;
1633}
1634
1635static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
1636{
1637 clear_bit_unlock(tag, &hba->lrb_in_use);
1638}
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
1650 enum dev_cmd_type cmd_type, int timeout)
1651{
1652 struct ufshcd_lrb *lrbp;
1653 int err;
1654 int tag;
1655 struct completion wait;
1656 unsigned long flags;
1657
1658
1659
1660
1661
1662
1663 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
1664
1665 init_completion(&wait);
1666 lrbp = &hba->lrb[tag];
1667 WARN_ON(lrbp->cmd);
1668 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
1669 if (unlikely(err))
1670 goto out_put_tag;
1671
1672 hba->dev_cmd.complete = &wait;
1673
1674
1675 wmb();
1676 spin_lock_irqsave(hba->host->host_lock, flags);
1677 ufshcd_send_command(hba, tag);
1678 spin_unlock_irqrestore(hba->host->host_lock, flags);
1679
1680 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
1681
1682out_put_tag:
1683 ufshcd_put_dev_cmd_tag(hba, tag);
1684 wake_up(&hba->dev_cmd.tag_wq);
1685 return err;
1686}
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698static inline void ufshcd_init_query(struct ufs_hba *hba,
1699 struct ufs_query_req **request, struct ufs_query_res **response,
1700 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
1701{
1702 *request = &hba->dev_cmd.query.request;
1703 *response = &hba->dev_cmd.query.response;
1704 memset(*request, 0, sizeof(struct ufs_query_req));
1705 memset(*response, 0, sizeof(struct ufs_query_res));
1706 (*request)->upiu_req.opcode = opcode;
1707 (*request)->upiu_req.idn = idn;
1708 (*request)->upiu_req.index = index;
1709 (*request)->upiu_req.selector = selector;
1710}
1711
1712static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1713 enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
1714{
1715 int ret;
1716 int retries;
1717
1718 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1719 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
1720 if (ret)
1721 dev_dbg(hba->dev,
1722 "%s: failed with error %d, retries %d\n",
1723 __func__, ret, retries);
1724 else
1725 break;
1726 }
1727
1728 if (ret)
1729 dev_err(hba->dev,
1730 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1731 __func__, opcode, idn, ret, retries);
1732 return ret;
1733}
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1745 enum flag_idn idn, bool *flag_res)
1746{
1747 struct ufs_query_req *request = NULL;
1748 struct ufs_query_res *response = NULL;
1749 int err, index = 0, selector = 0;
1750 int timeout = QUERY_REQ_TIMEOUT;
1751
1752 BUG_ON(!hba);
1753
1754 ufshcd_hold(hba, false);
1755 mutex_lock(&hba->dev_cmd.lock);
1756 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1757 selector);
1758
1759 switch (opcode) {
1760 case UPIU_QUERY_OPCODE_SET_FLAG:
1761 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1762 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1763 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1764 break;
1765 case UPIU_QUERY_OPCODE_READ_FLAG:
1766 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1767 if (!flag_res) {
1768
1769 dev_err(hba->dev, "%s: Invalid argument for read request\n",
1770 __func__);
1771 err = -EINVAL;
1772 goto out_unlock;
1773 }
1774 break;
1775 default:
1776 dev_err(hba->dev,
1777 "%s: Expected query flag opcode but got = %d\n",
1778 __func__, opcode);
1779 err = -EINVAL;
1780 goto out_unlock;
1781 }
1782
1783 if (idn == QUERY_FLAG_IDN_FDEVICEINIT)
1784 timeout = QUERY_FDEVICEINIT_REQ_TIMEOUT;
1785
1786 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
1787
1788 if (err) {
1789 dev_err(hba->dev,
1790 "%s: Sending flag query for idn %d failed, err = %d\n",
1791 __func__, idn, err);
1792 goto out_unlock;
1793 }
1794
1795 if (flag_res)
1796 *flag_res = (be32_to_cpu(response->upiu_res.value) &
1797 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1798
1799out_unlock:
1800 mutex_unlock(&hba->dev_cmd.lock);
1801 ufshcd_release(hba);
1802 return err;
1803}
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1817 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
1818{
1819 struct ufs_query_req *request = NULL;
1820 struct ufs_query_res *response = NULL;
1821 int err;
1822
1823 BUG_ON(!hba);
1824
1825 ufshcd_hold(hba, false);
1826 if (!attr_val) {
1827 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
1828 __func__, opcode);
1829 err = -EINVAL;
1830 goto out;
1831 }
1832
1833 mutex_lock(&hba->dev_cmd.lock);
1834 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1835 selector);
1836
1837 switch (opcode) {
1838 case UPIU_QUERY_OPCODE_WRITE_ATTR:
1839 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1840 request->upiu_req.value = cpu_to_be32(*attr_val);
1841 break;
1842 case UPIU_QUERY_OPCODE_READ_ATTR:
1843 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1844 break;
1845 default:
1846 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
1847 __func__, opcode);
1848 err = -EINVAL;
1849 goto out_unlock;
1850 }
1851
1852 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1853
1854 if (err) {
1855 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1856 __func__, opcode, idn, err);
1857 goto out_unlock;
1858 }
1859
1860 *attr_val = be32_to_cpu(response->upiu_res.value);
1861
1862out_unlock:
1863 mutex_unlock(&hba->dev_cmd.lock);
1864out:
1865 ufshcd_release(hba);
1866 return err;
1867}
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882static int ufshcd_query_attr_retry(struct ufs_hba *hba,
1883 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
1884 u32 *attr_val)
1885{
1886 int ret = 0;
1887 u32 retries;
1888
1889 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1890 ret = ufshcd_query_attr(hba, opcode, idn, index,
1891 selector, attr_val);
1892 if (ret)
1893 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
1894 __func__, ret, retries);
1895 else
1896 break;
1897 }
1898
1899 if (ret)
1900 dev_err(hba->dev,
1901 "%s: query attribute, idn %d, failed with error %d after %d retires\n",
1902 __func__, idn, ret, QUERY_REQ_RETRIES);
1903 return ret;
1904}
1905
1906static int __ufshcd_query_descriptor(struct ufs_hba *hba,
1907 enum query_opcode opcode, enum desc_idn idn, u8 index,
1908 u8 selector, u8 *desc_buf, int *buf_len)
1909{
1910 struct ufs_query_req *request = NULL;
1911 struct ufs_query_res *response = NULL;
1912 int err;
1913
1914 BUG_ON(!hba);
1915
1916 ufshcd_hold(hba, false);
1917 if (!desc_buf) {
1918 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
1919 __func__, opcode);
1920 err = -EINVAL;
1921 goto out;
1922 }
1923
1924 if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
1925 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
1926 __func__, *buf_len);
1927 err = -EINVAL;
1928 goto out;
1929 }
1930
1931 mutex_lock(&hba->dev_cmd.lock);
1932 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1933 selector);
1934 hba->dev_cmd.query.descriptor = desc_buf;
1935 request->upiu_req.length = cpu_to_be16(*buf_len);
1936
1937 switch (opcode) {
1938 case UPIU_QUERY_OPCODE_WRITE_DESC:
1939 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1940 break;
1941 case UPIU_QUERY_OPCODE_READ_DESC:
1942 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1943 break;
1944 default:
1945 dev_err(hba->dev,
1946 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1947 __func__, opcode);
1948 err = -EINVAL;
1949 goto out_unlock;
1950 }
1951
1952 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1953
1954 if (err) {
1955 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1956 __func__, opcode, idn, err);
1957 goto out_unlock;
1958 }
1959
1960 hba->dev_cmd.query.descriptor = NULL;
1961 *buf_len = be16_to_cpu(response->upiu_res.length);
1962
1963out_unlock:
1964 mutex_unlock(&hba->dev_cmd.lock);
1965out:
1966 ufshcd_release(hba);
1967 return err;
1968}
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
1986 enum query_opcode opcode, enum desc_idn idn, u8 index,
1987 u8 selector, u8 *desc_buf, int *buf_len)
1988{
1989 int err;
1990 int retries;
1991
1992 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1993 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
1994 selector, desc_buf, buf_len);
1995 if (!err || err == -EINVAL)
1996 break;
1997 }
1998
1999 return err;
2000}
2001EXPORT_SYMBOL(ufshcd_query_descriptor_retry);
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014static int ufshcd_read_desc_param(struct ufs_hba *hba,
2015 enum desc_idn desc_id,
2016 int desc_index,
2017 u32 param_offset,
2018 u8 *param_read_buf,
2019 u32 param_size)
2020{
2021 int ret;
2022 u8 *desc_buf;
2023 u32 buff_len;
2024 bool is_kmalloc = true;
2025
2026
2027 if (desc_id >= QUERY_DESC_IDN_MAX)
2028 return -EINVAL;
2029
2030 buff_len = ufs_query_desc_max_size[desc_id];
2031 if ((param_offset + param_size) > buff_len)
2032 return -EINVAL;
2033
2034 if (!param_offset && (param_size == buff_len)) {
2035
2036 desc_buf = param_read_buf;
2037 is_kmalloc = false;
2038 } else {
2039
2040 desc_buf = kmalloc(buff_len, GFP_KERNEL);
2041 if (!desc_buf)
2042 return -ENOMEM;
2043 }
2044
2045 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
2046 desc_id, desc_index, 0, desc_buf,
2047 &buff_len);
2048
2049 if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) ||
2050 (desc_buf[QUERY_DESC_LENGTH_OFFSET] !=
2051 ufs_query_desc_max_size[desc_id])
2052 || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) {
2053 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d",
2054 __func__, desc_id, param_offset, buff_len, ret);
2055 if (!ret)
2056 ret = -EINVAL;
2057
2058 goto out;
2059 }
2060
2061 if (is_kmalloc)
2062 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
2063out:
2064 if (is_kmalloc)
2065 kfree(desc_buf);
2066 return ret;
2067}
2068
2069static inline int ufshcd_read_desc(struct ufs_hba *hba,
2070 enum desc_idn desc_id,
2071 int desc_index,
2072 u8 *buf,
2073 u32 size)
2074{
2075 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
2076}
2077
2078static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
2079 u8 *buf,
2080 u32 size)
2081{
2082 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
2083}
2084
2085int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
2086{
2087 return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
2088}
2089EXPORT_SYMBOL(ufshcd_read_device_desc);
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, u8 *buf,
2102 u32 size, bool ascii)
2103{
2104 int err = 0;
2105
2106 err = ufshcd_read_desc(hba,
2107 QUERY_DESC_IDN_STRING, desc_index, buf, size);
2108
2109 if (err) {
2110 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
2111 __func__, QUERY_REQ_RETRIES, err);
2112 goto out;
2113 }
2114
2115 if (ascii) {
2116 int desc_len;
2117 int ascii_len;
2118 int i;
2119 char *buff_ascii;
2120
2121 desc_len = buf[0];
2122
2123 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
2124 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
2125 dev_err(hba->dev, "%s: buffer allocated size is too small\n",
2126 __func__);
2127 err = -ENOMEM;
2128 goto out;
2129 }
2130
2131 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
2132 if (!buff_ascii) {
2133 err = -ENOMEM;
2134 goto out_free_buff;
2135 }
2136
2137
2138
2139
2140
2141 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
2142 desc_len - QUERY_DESC_HDR_SIZE,
2143 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
2144
2145
2146 for (i = 0; i < ascii_len; i++)
2147 ufshcd_remove_non_printable(&buff_ascii[i]);
2148
2149 memset(buf + QUERY_DESC_HDR_SIZE, 0,
2150 size - QUERY_DESC_HDR_SIZE);
2151 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
2152 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
2153out_free_buff:
2154 kfree(buff_ascii);
2155 }
2156out:
2157 return err;
2158}
2159EXPORT_SYMBOL(ufshcd_read_string_desc);
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
2172 int lun,
2173 enum unit_desc_param param_offset,
2174 u8 *param_read_buf,
2175 u32 param_size)
2176{
2177
2178
2179
2180
2181 if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN))
2182 return -EOPNOTSUPP;
2183
2184 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
2185 param_offset, param_read_buf, param_size);
2186}
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201static int ufshcd_memory_alloc(struct ufs_hba *hba)
2202{
2203 size_t utmrdl_size, utrdl_size, ucdl_size;
2204
2205
2206 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2207 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
2208 ucdl_size,
2209 &hba->ucdl_dma_addr,
2210 GFP_KERNEL);
2211
2212
2213
2214
2215
2216
2217
2218 if (!hba->ucdl_base_addr ||
2219 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
2220 dev_err(hba->dev,
2221 "Command Descriptor Memory allocation failed\n");
2222 goto out;
2223 }
2224
2225
2226
2227
2228
2229 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2230 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
2231 utrdl_size,
2232 &hba->utrdl_dma_addr,
2233 GFP_KERNEL);
2234 if (!hba->utrdl_base_addr ||
2235 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
2236 dev_err(hba->dev,
2237 "Transfer Descriptor Memory allocation failed\n");
2238 goto out;
2239 }
2240
2241
2242
2243
2244
2245 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2246 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
2247 utmrdl_size,
2248 &hba->utmrdl_dma_addr,
2249 GFP_KERNEL);
2250 if (!hba->utmrdl_base_addr ||
2251 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
2252 dev_err(hba->dev,
2253 "Task Management Descriptor Memory allocation failed\n");
2254 goto out;
2255 }
2256
2257
2258 hba->lrb = devm_kzalloc(hba->dev,
2259 hba->nutrs * sizeof(struct ufshcd_lrb),
2260 GFP_KERNEL);
2261 if (!hba->lrb) {
2262 dev_err(hba->dev, "LRB Memory allocation failed\n");
2263 goto out;
2264 }
2265 return 0;
2266out:
2267 return -ENOMEM;
2268}
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283static void ufshcd_host_memory_configure(struct ufs_hba *hba)
2284{
2285 struct utp_transfer_cmd_desc *cmd_descp;
2286 struct utp_transfer_req_desc *utrdlp;
2287 dma_addr_t cmd_desc_dma_addr;
2288 dma_addr_t cmd_desc_element_addr;
2289 u16 response_offset;
2290 u16 prdt_offset;
2291 int cmd_desc_size;
2292 int i;
2293
2294 utrdlp = hba->utrdl_base_addr;
2295 cmd_descp = hba->ucdl_base_addr;
2296
2297 response_offset =
2298 offsetof(struct utp_transfer_cmd_desc, response_upiu);
2299 prdt_offset =
2300 offsetof(struct utp_transfer_cmd_desc, prd_table);
2301
2302 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
2303 cmd_desc_dma_addr = hba->ucdl_dma_addr;
2304
2305 for (i = 0; i < hba->nutrs; i++) {
2306
2307 cmd_desc_element_addr =
2308 (cmd_desc_dma_addr + (cmd_desc_size * i));
2309 utrdlp[i].command_desc_base_addr_lo =
2310 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
2311 utrdlp[i].command_desc_base_addr_hi =
2312 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
2313
2314
2315 utrdlp[i].response_upiu_offset =
2316 cpu_to_le16((response_offset >> 2));
2317 utrdlp[i].prd_table_offset =
2318 cpu_to_le16((prdt_offset >> 2));
2319 utrdlp[i].response_upiu_length =
2320 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
2321
2322 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
2323 hba->lrb[i].ucd_req_ptr =
2324 (struct utp_upiu_req *)(cmd_descp + i);
2325 hba->lrb[i].ucd_rsp_ptr =
2326 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2327 hba->lrb[i].ucd_prdt_ptr =
2328 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2329 }
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343static int ufshcd_dme_link_startup(struct ufs_hba *hba)
2344{
2345 struct uic_command uic_cmd = {0};
2346 int ret;
2347
2348 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
2349
2350 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2351 if (ret)
2352 dev_err(hba->dev,
2353 "dme-link-startup: error code %d\n", ret);
2354 return ret;
2355}
2356
2357static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
2358{
2359 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
2360 unsigned long min_sleep_time_us;
2361
2362 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
2363 return;
2364
2365
2366
2367
2368
2369 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
2370 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
2371 } else {
2372 unsigned long delta =
2373 (unsigned long) ktime_to_us(
2374 ktime_sub(ktime_get(),
2375 hba->last_dme_cmd_tstamp));
2376
2377 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
2378 min_sleep_time_us =
2379 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
2380 else
2381 return;
2382 }
2383
2384
2385 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
2386}
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
2399 u8 attr_set, u32 mib_val, u8 peer)
2400{
2401 struct uic_command uic_cmd = {0};
2402 static const char *const action[] = {
2403 "dme-set",
2404 "dme-peer-set"
2405 };
2406 const char *set = action[!!peer];
2407 int ret;
2408 int retries = UFS_UIC_COMMAND_RETRIES;
2409
2410 uic_cmd.command = peer ?
2411 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
2412 uic_cmd.argument1 = attr_sel;
2413 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
2414 uic_cmd.argument3 = mib_val;
2415
2416 do {
2417
2418 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2419 if (ret)
2420 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
2421 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
2422 } while (ret && peer && --retries);
2423
2424 if (!retries)
2425 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
2426 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
2427 retries);
2428
2429 return ret;
2430}
2431EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
2443 u32 *mib_val, u8 peer)
2444{
2445 struct uic_command uic_cmd = {0};
2446 static const char *const action[] = {
2447 "dme-get",
2448 "dme-peer-get"
2449 };
2450 const char *get = action[!!peer];
2451 int ret;
2452 int retries = UFS_UIC_COMMAND_RETRIES;
2453 struct ufs_pa_layer_attr orig_pwr_info;
2454 struct ufs_pa_layer_attr temp_pwr_info;
2455 bool pwr_mode_change = false;
2456
2457 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
2458 orig_pwr_info = hba->pwr_info;
2459 temp_pwr_info = orig_pwr_info;
2460
2461 if (orig_pwr_info.pwr_tx == FAST_MODE ||
2462 orig_pwr_info.pwr_rx == FAST_MODE) {
2463 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
2464 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
2465 pwr_mode_change = true;
2466 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
2467 orig_pwr_info.pwr_rx == SLOW_MODE) {
2468 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
2469 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
2470 pwr_mode_change = true;
2471 }
2472 if (pwr_mode_change) {
2473 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
2474 if (ret)
2475 goto out;
2476 }
2477 }
2478
2479 uic_cmd.command = peer ?
2480 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
2481 uic_cmd.argument1 = attr_sel;
2482
2483 do {
2484
2485 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2486 if (ret)
2487 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
2488 get, UIC_GET_ATTR_ID(attr_sel), ret);
2489 } while (ret && peer && --retries);
2490
2491 if (!retries)
2492 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
2493 get, UIC_GET_ATTR_ID(attr_sel), retries);
2494
2495 if (mib_val && !ret)
2496 *mib_val = uic_cmd.argument3;
2497
2498 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
2499 && pwr_mode_change)
2500 ufshcd_change_power_mode(hba, &orig_pwr_info);
2501out:
2502 return ret;
2503}
2504EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
2523{
2524 struct completion uic_async_done;
2525 unsigned long flags;
2526 u8 status;
2527 int ret;
2528 bool reenable_intr = false;
2529
2530 mutex_lock(&hba->uic_cmd_mutex);
2531 init_completion(&uic_async_done);
2532 ufshcd_add_delay_before_dme_cmd(hba);
2533
2534 spin_lock_irqsave(hba->host->host_lock, flags);
2535 hba->uic_async_done = &uic_async_done;
2536 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
2537 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
2538
2539
2540
2541
2542 wmb();
2543 reenable_intr = true;
2544 }
2545 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
2546 spin_unlock_irqrestore(hba->host->host_lock, flags);
2547 if (ret) {
2548 dev_err(hba->dev,
2549 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
2550 cmd->command, cmd->argument3, ret);
2551 goto out;
2552 }
2553
2554 if (!wait_for_completion_timeout(hba->uic_async_done,
2555 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2556 dev_err(hba->dev,
2557 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
2558 cmd->command, cmd->argument3);
2559 ret = -ETIMEDOUT;
2560 goto out;
2561 }
2562
2563 status = ufshcd_get_upmcrs(hba);
2564 if (status != PWR_LOCAL) {
2565 dev_err(hba->dev,
2566 "pwr ctrl cmd 0x%0x failed, host umpcrs:0x%x\n",
2567 cmd->command, status);
2568 ret = (status != PWR_OK) ? status : -1;
2569 }
2570out:
2571 spin_lock_irqsave(hba->host->host_lock, flags);
2572 hba->active_uic_cmd = NULL;
2573 hba->uic_async_done = NULL;
2574 if (reenable_intr)
2575 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
2576 spin_unlock_irqrestore(hba->host->host_lock, flags);
2577 mutex_unlock(&hba->uic_cmd_mutex);
2578
2579 return ret;
2580}
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
2591{
2592 struct uic_command uic_cmd = {0};
2593 int ret;
2594
2595 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
2596 ret = ufshcd_dme_set(hba,
2597 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
2598 if (ret) {
2599 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
2600 __func__, ret);
2601 goto out;
2602 }
2603 }
2604
2605 uic_cmd.command = UIC_CMD_DME_SET;
2606 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
2607 uic_cmd.argument3 = mode;
2608 ufshcd_hold(hba, false);
2609 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2610 ufshcd_release(hba);
2611
2612out:
2613 return ret;
2614}
2615
2616static int ufshcd_link_recovery(struct ufs_hba *hba)
2617{
2618 int ret;
2619 unsigned long flags;
2620
2621 spin_lock_irqsave(hba->host->host_lock, flags);
2622 hba->ufshcd_state = UFSHCD_STATE_RESET;
2623 ufshcd_set_eh_in_progress(hba);
2624 spin_unlock_irqrestore(hba->host->host_lock, flags);
2625
2626 ret = ufshcd_host_reset_and_restore(hba);
2627
2628 spin_lock_irqsave(hba->host->host_lock, flags);
2629 if (ret)
2630 hba->ufshcd_state = UFSHCD_STATE_ERROR;
2631 ufshcd_clear_eh_in_progress(hba);
2632 spin_unlock_irqrestore(hba->host->host_lock, flags);
2633
2634 if (ret)
2635 dev_err(hba->dev, "%s: link recovery failed, err %d",
2636 __func__, ret);
2637
2638 return ret;
2639}
2640
2641static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2642{
2643 int ret;
2644 struct uic_command uic_cmd = {0};
2645
2646 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
2647 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2648
2649 if (ret) {
2650 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
2651 __func__, ret);
2652
2653
2654
2655
2656
2657 if (ufshcd_link_recovery(hba))
2658 ret = -ENOLINK;
2659 }
2660
2661 return ret;
2662}
2663
2664static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2665{
2666 int ret = 0, retries;
2667
2668 for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
2669 ret = __ufshcd_uic_hibern8_enter(hba);
2670 if (!ret || ret == -ENOLINK)
2671 goto out;
2672 }
2673out:
2674 return ret;
2675}
2676
2677static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
2678{
2679 struct uic_command uic_cmd = {0};
2680 int ret;
2681
2682 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
2683 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2684 if (ret) {
2685 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
2686 __func__, ret);
2687 ret = ufshcd_link_recovery(hba);
2688 }
2689
2690 return ret;
2691}
2692
2693
2694
2695
2696
2697
2698static void ufshcd_init_pwr_info(struct ufs_hba *hba)
2699{
2700 hba->pwr_info.gear_rx = UFS_PWM_G1;
2701 hba->pwr_info.gear_tx = UFS_PWM_G1;
2702 hba->pwr_info.lane_rx = 1;
2703 hba->pwr_info.lane_tx = 1;
2704 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
2705 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
2706 hba->pwr_info.hs_rate = 0;
2707}
2708
2709
2710
2711
2712
2713static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
2714{
2715 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
2716
2717 if (hba->max_pwr_info.is_valid)
2718 return 0;
2719
2720 pwr_info->pwr_tx = FASTAUTO_MODE;
2721 pwr_info->pwr_rx = FASTAUTO_MODE;
2722 pwr_info->hs_rate = PA_HS_MODE_B;
2723
2724
2725 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
2726 &pwr_info->lane_rx);
2727 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
2728 &pwr_info->lane_tx);
2729
2730 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
2731 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
2732 __func__,
2733 pwr_info->lane_rx,
2734 pwr_info->lane_tx);
2735 return -EINVAL;
2736 }
2737
2738
2739
2740
2741
2742
2743 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
2744 if (!pwr_info->gear_rx) {
2745 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2746 &pwr_info->gear_rx);
2747 if (!pwr_info->gear_rx) {
2748 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
2749 __func__, pwr_info->gear_rx);
2750 return -EINVAL;
2751 }
2752 pwr_info->pwr_rx = SLOWAUTO_MODE;
2753 }
2754
2755 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
2756 &pwr_info->gear_tx);
2757 if (!pwr_info->gear_tx) {
2758 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2759 &pwr_info->gear_tx);
2760 if (!pwr_info->gear_tx) {
2761 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
2762 __func__, pwr_info->gear_tx);
2763 return -EINVAL;
2764 }
2765 pwr_info->pwr_tx = SLOWAUTO_MODE;
2766 }
2767
2768 hba->max_pwr_info.is_valid = true;
2769 return 0;
2770}
2771
2772static int ufshcd_change_power_mode(struct ufs_hba *hba,
2773 struct ufs_pa_layer_attr *pwr_mode)
2774{
2775 int ret;
2776
2777
2778 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
2779 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
2780 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
2781 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
2782 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
2783 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
2784 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
2785 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
2786 return 0;
2787 }
2788
2789
2790
2791
2792
2793
2794
2795 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
2796 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
2797 pwr_mode->lane_rx);
2798 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2799 pwr_mode->pwr_rx == FAST_MODE)
2800 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
2801 else
2802 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
2803
2804 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
2805 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
2806 pwr_mode->lane_tx);
2807 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
2808 pwr_mode->pwr_tx == FAST_MODE)
2809 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
2810 else
2811 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
2812
2813 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2814 pwr_mode->pwr_tx == FASTAUTO_MODE ||
2815 pwr_mode->pwr_rx == FAST_MODE ||
2816 pwr_mode->pwr_tx == FAST_MODE)
2817 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
2818 pwr_mode->hs_rate);
2819
2820 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
2821 | pwr_mode->pwr_tx);
2822
2823 if (ret) {
2824 dev_err(hba->dev,
2825 "%s: power mode change failed %d\n", __func__, ret);
2826 } else {
2827 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
2828 pwr_mode);
2829
2830 memcpy(&hba->pwr_info, pwr_mode,
2831 sizeof(struct ufs_pa_layer_attr));
2832 }
2833
2834 return ret;
2835}
2836
2837
2838
2839
2840
2841
2842static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
2843 struct ufs_pa_layer_attr *desired_pwr_mode)
2844{
2845 struct ufs_pa_layer_attr final_params = { 0 };
2846 int ret;
2847
2848 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
2849 desired_pwr_mode, &final_params);
2850
2851 if (ret)
2852 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
2853
2854 ret = ufshcd_change_power_mode(hba, &final_params);
2855
2856 return ret;
2857}
2858
2859
2860
2861
2862
2863
2864
2865static int ufshcd_complete_dev_init(struct ufs_hba *hba)
2866{
2867 int i;
2868 int err;
2869 bool flag_res = 1;
2870
2871 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
2872 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
2873 if (err) {
2874 dev_err(hba->dev,
2875 "%s setting fDeviceInit flag failed with error %d\n",
2876 __func__, err);
2877 goto out;
2878 }
2879
2880
2881 for (i = 0; i < 1000 && !err && flag_res; i++)
2882 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
2883 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
2884
2885 if (err)
2886 dev_err(hba->dev,
2887 "%s reading fDeviceInit flag failed with error %d\n",
2888 __func__, err);
2889 else if (flag_res)
2890 dev_err(hba->dev,
2891 "%s fDeviceInit was not cleared by the device\n",
2892 __func__);
2893
2894out:
2895 return err;
2896}
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910static int ufshcd_make_hba_operational(struct ufs_hba *hba)
2911{
2912 int err = 0;
2913 u32 reg;
2914
2915
2916 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
2917
2918
2919 if (ufshcd_is_intr_aggr_allowed(hba))
2920 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
2921 else
2922 ufshcd_disable_intr_aggr(hba);
2923
2924
2925 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
2926 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
2927 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
2928 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
2929 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
2930 REG_UTP_TASK_REQ_LIST_BASE_L);
2931 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
2932 REG_UTP_TASK_REQ_LIST_BASE_H);
2933
2934
2935
2936
2937
2938 wmb();
2939
2940
2941
2942
2943 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
2944 if (!(ufshcd_get_lists_status(reg))) {
2945 ufshcd_enable_run_stop_reg(hba);
2946 } else {
2947 dev_err(hba->dev,
2948 "Host controller not ready to process requests");
2949 err = -EIO;
2950 goto out;
2951 }
2952
2953out:
2954 return err;
2955}
2956
2957
2958
2959
2960
2961
2962static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
2963{
2964 int err;
2965
2966 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
2967 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
2968 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
2969 10, 1, can_sleep);
2970 if (err)
2971 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
2972}
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984static int ufshcd_hba_enable(struct ufs_hba *hba)
2985{
2986 int retry;
2987
2988
2989
2990
2991
2992
2993
2994 if (!ufshcd_is_hba_active(hba))
2995
2996 ufshcd_hba_stop(hba, true);
2997
2998
2999 ufshcd_set_link_off(hba);
3000
3001 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
3002
3003
3004 ufshcd_hba_start(hba);
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016 msleep(1);
3017
3018
3019 retry = 10;
3020 while (ufshcd_is_hba_active(hba)) {
3021 if (retry) {
3022 retry--;
3023 } else {
3024 dev_err(hba->dev,
3025 "Controller enable failed\n");
3026 return -EIO;
3027 }
3028 msleep(5);
3029 }
3030
3031
3032 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
3033
3034 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
3035
3036 return 0;
3037}
3038
3039static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
3040{
3041 int tx_lanes, i, err = 0;
3042
3043 if (!peer)
3044 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3045 &tx_lanes);
3046 else
3047 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3048 &tx_lanes);
3049 for (i = 0; i < tx_lanes; i++) {
3050 if (!peer)
3051 err = ufshcd_dme_set(hba,
3052 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3053 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3054 0);
3055 else
3056 err = ufshcd_dme_peer_set(hba,
3057 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3058 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3059 0);
3060 if (err) {
3061 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
3062 __func__, peer, i, err);
3063 break;
3064 }
3065 }
3066
3067 return err;
3068}
3069
3070static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
3071{
3072 return ufshcd_disable_tx_lcc(hba, true);
3073}
3074
3075
3076
3077
3078
3079
3080
3081static int ufshcd_link_startup(struct ufs_hba *hba)
3082{
3083 int ret;
3084 int retries = DME_LINKSTARTUP_RETRIES;
3085
3086 do {
3087 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
3088
3089 ret = ufshcd_dme_link_startup(hba);
3090
3091
3092 if (!ret && !ufshcd_is_device_present(hba)) {
3093 dev_err(hba->dev, "%s: Device not present\n", __func__);
3094 ret = -ENXIO;
3095 goto out;
3096 }
3097
3098
3099
3100
3101
3102
3103 if (ret && ufshcd_hba_enable(hba))
3104 goto out;
3105 } while (ret && retries--);
3106
3107 if (ret)
3108
3109 goto out;
3110
3111 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
3112 ret = ufshcd_disable_device_tx_lcc(hba);
3113 if (ret)
3114 goto out;
3115 }
3116
3117
3118 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
3119 if (ret)
3120 goto out;
3121
3122 ret = ufshcd_make_hba_operational(hba);
3123out:
3124 if (ret)
3125 dev_err(hba->dev, "link startup failed %d\n", ret);
3126 return ret;
3127}
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139static int ufshcd_verify_dev_init(struct ufs_hba *hba)
3140{
3141 int err = 0;
3142 int retries;
3143
3144 ufshcd_hold(hba, false);
3145 mutex_lock(&hba->dev_cmd.lock);
3146 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
3147 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
3148 NOP_OUT_TIMEOUT);
3149
3150 if (!err || err == -ETIMEDOUT)
3151 break;
3152
3153 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
3154 }
3155 mutex_unlock(&hba->dev_cmd.lock);
3156 ufshcd_release(hba);
3157
3158 if (err)
3159 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
3160 return err;
3161}
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172static void ufshcd_set_queue_depth(struct scsi_device *sdev)
3173{
3174 int ret = 0;
3175 u8 lun_qdepth;
3176 struct ufs_hba *hba;
3177
3178 hba = shost_priv(sdev->host);
3179
3180 lun_qdepth = hba->nutrs;
3181 ret = ufshcd_read_unit_desc_param(hba,
3182 ufshcd_scsi_to_upiu_lun(sdev->lun),
3183 UNIT_DESC_PARAM_LU_Q_DEPTH,
3184 &lun_qdepth,
3185 sizeof(lun_qdepth));
3186
3187
3188 if (ret == -EOPNOTSUPP)
3189 lun_qdepth = 1;
3190 else if (!lun_qdepth)
3191
3192 lun_qdepth = hba->nutrs;
3193 else
3194 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
3195
3196 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
3197 __func__, lun_qdepth);
3198 scsi_change_queue_depth(sdev, lun_qdepth);
3199}
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212static int ufshcd_get_lu_wp(struct ufs_hba *hba,
3213 u8 lun,
3214 u8 *b_lu_write_protect)
3215{
3216 int ret;
3217
3218 if (!b_lu_write_protect)
3219 ret = -EINVAL;
3220
3221
3222
3223
3224
3225 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
3226 ret = -ENOTSUPP;
3227 else
3228 ret = ufshcd_read_unit_desc_param(hba,
3229 lun,
3230 UNIT_DESC_PARAM_LU_WR_PROTECT,
3231 b_lu_write_protect,
3232 sizeof(*b_lu_write_protect));
3233 return ret;
3234}
3235
3236
3237
3238
3239
3240
3241
3242
3243static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
3244 struct scsi_device *sdev)
3245{
3246 if (hba->dev_info.f_power_on_wp_en &&
3247 !hba->dev_info.is_lu_power_on_wp) {
3248 u8 b_lu_write_protect;
3249
3250 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
3251 &b_lu_write_protect) &&
3252 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
3253 hba->dev_info.is_lu_power_on_wp = true;
3254 }
3255}
3256
3257
3258
3259
3260
3261
3262
3263static int ufshcd_slave_alloc(struct scsi_device *sdev)
3264{
3265 struct ufs_hba *hba;
3266
3267 hba = shost_priv(sdev->host);
3268
3269
3270 sdev->use_10_for_ms = 1;
3271
3272
3273 sdev->allow_restart = 1;
3274
3275
3276 sdev->no_report_opcodes = 1;
3277
3278
3279 ufshcd_set_queue_depth(sdev);
3280
3281 ufshcd_get_lu_power_on_wp_status(hba, sdev);
3282
3283 return 0;
3284}
3285
3286
3287
3288
3289
3290
3291
3292
3293static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
3294{
3295 struct ufs_hba *hba = shost_priv(sdev->host);
3296
3297 if (depth > hba->nutrs)
3298 depth = hba->nutrs;
3299 return scsi_change_queue_depth(sdev, depth);
3300}
3301
3302
3303
3304
3305
3306static int ufshcd_slave_configure(struct scsi_device *sdev)
3307{
3308 struct request_queue *q = sdev->request_queue;
3309
3310 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
3311 blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX);
3312
3313 return 0;
3314}
3315
3316
3317
3318
3319
3320static void ufshcd_slave_destroy(struct scsi_device *sdev)
3321{
3322 struct ufs_hba *hba;
3323
3324 hba = shost_priv(sdev->host);
3325
3326 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
3327 unsigned long flags;
3328
3329 spin_lock_irqsave(hba->host->host_lock, flags);
3330 hba->sdev_ufs_device = NULL;
3331 spin_unlock_irqrestore(hba->host->host_lock, flags);
3332 }
3333}
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
3344{
3345 struct utp_task_req_desc *task_req_descp;
3346 struct utp_upiu_task_rsp *task_rsp_upiup;
3347 unsigned long flags;
3348 int ocs_value;
3349 int task_result;
3350
3351 spin_lock_irqsave(hba->host->host_lock, flags);
3352
3353
3354 __clear_bit(index, &hba->outstanding_tasks);
3355
3356 task_req_descp = hba->utmrdl_base_addr;
3357 ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
3358
3359 if (ocs_value == OCS_SUCCESS) {
3360 task_rsp_upiup = (struct utp_upiu_task_rsp *)
3361 task_req_descp[index].task_rsp_upiu;
3362 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
3363 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
3364 if (resp)
3365 *resp = (u8)task_result;
3366 } else {
3367 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
3368 __func__, ocs_value);
3369 }
3370 spin_unlock_irqrestore(hba->host->host_lock, flags);
3371
3372 return ocs_value;
3373}
3374
3375
3376
3377
3378
3379
3380
3381
3382static inline int
3383ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
3384{
3385 int result = 0;
3386
3387 switch (scsi_status) {
3388 case SAM_STAT_CHECK_CONDITION:
3389 ufshcd_copy_sense_data(lrbp);
3390 case SAM_STAT_GOOD:
3391 result |= DID_OK << 16 |
3392 COMMAND_COMPLETE << 8 |
3393 scsi_status;
3394 break;
3395 case SAM_STAT_TASK_SET_FULL:
3396 case SAM_STAT_BUSY:
3397 case SAM_STAT_TASK_ABORTED:
3398 ufshcd_copy_sense_data(lrbp);
3399 result |= scsi_status;
3400 break;
3401 default:
3402 result |= DID_ERROR << 16;
3403 break;
3404 }
3405
3406 return result;
3407}
3408
3409
3410
3411
3412
3413
3414
3415
3416static inline int
3417ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3418{
3419 int result = 0;
3420 int scsi_status;
3421 int ocs;
3422
3423
3424 ocs = ufshcd_get_tr_ocs(lrbp);
3425
3426 switch (ocs) {
3427 case OCS_SUCCESS:
3428 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3429
3430 switch (result) {
3431 case UPIU_TRANSACTION_RESPONSE:
3432
3433
3434
3435
3436 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
3437
3438
3439
3440
3441
3442 scsi_status = result & MASK_SCSI_STATUS;
3443 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457 if (!hba->pm_op_in_progress &&
3458 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
3459 schedule_work(&hba->eeh_work);
3460 break;
3461 case UPIU_TRANSACTION_REJECT_UPIU:
3462
3463 result = DID_ERROR << 16;
3464 dev_err(hba->dev,
3465 "Reject UPIU not fully implemented\n");
3466 break;
3467 default:
3468 result = DID_ERROR << 16;
3469 dev_err(hba->dev,
3470 "Unexpected request response code = %x\n",
3471 result);
3472 break;
3473 }
3474 break;
3475 case OCS_ABORTED:
3476 result |= DID_ABORT << 16;
3477 break;
3478 case OCS_INVALID_COMMAND_STATUS:
3479 result |= DID_REQUEUE << 16;
3480 break;
3481 case OCS_INVALID_CMD_TABLE_ATTR:
3482 case OCS_INVALID_PRDT_ATTR:
3483 case OCS_MISMATCH_DATA_BUF_SIZE:
3484 case OCS_MISMATCH_RESP_UPIU_SIZE:
3485 case OCS_PEER_COMM_FAILURE:
3486 case OCS_FATAL_ERROR:
3487 default:
3488 result |= DID_ERROR << 16;
3489 dev_err(hba->dev,
3490 "OCS error from controller = %x\n", ocs);
3491 break;
3492 }
3493
3494 return result;
3495}
3496
3497
3498
3499
3500
3501
3502static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
3503{
3504 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
3505 hba->active_uic_cmd->argument2 |=
3506 ufshcd_get_uic_cmd_result(hba);
3507 hba->active_uic_cmd->argument3 =
3508 ufshcd_get_dme_attr_val(hba);
3509 complete(&hba->active_uic_cmd->done);
3510 }
3511
3512 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done)
3513 complete(hba->uic_async_done);
3514}
3515
3516
3517
3518
3519
3520
3521static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
3522 unsigned long completed_reqs)
3523{
3524 struct ufshcd_lrb *lrbp;
3525 struct scsi_cmnd *cmd;
3526 int result;
3527 int index;
3528
3529 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
3530 lrbp = &hba->lrb[index];
3531 cmd = lrbp->cmd;
3532 if (cmd) {
3533 result = ufshcd_transfer_rsp_status(hba, lrbp);
3534 scsi_dma_unmap(cmd);
3535 cmd->result = result;
3536
3537 lrbp->cmd = NULL;
3538 clear_bit_unlock(index, &hba->lrb_in_use);
3539
3540 cmd->scsi_done(cmd);
3541 __ufshcd_release(hba);
3542 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) {
3543 if (hba->dev_cmd.complete)
3544 complete(hba->dev_cmd.complete);
3545 }
3546 }
3547
3548
3549 hba->outstanding_reqs ^= completed_reqs;
3550
3551 ufshcd_clk_scaling_update_busy(hba);
3552
3553
3554 wake_up(&hba->dev_cmd.tag_wq);
3555}
3556
3557
3558
3559
3560
3561static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
3562{
3563 unsigned long completed_reqs;
3564 u32 tr_doorbell;
3565
3566
3567
3568
3569
3570
3571
3572
3573 if (ufshcd_is_intr_aggr_allowed(hba))
3574 ufshcd_reset_intr_aggr(hba);
3575
3576 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
3577 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
3578
3579 __ufshcd_transfer_req_compl(hba, completed_reqs);
3580}
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
3593{
3594 int err = 0;
3595 u32 val;
3596
3597 if (!(hba->ee_ctrl_mask & mask))
3598 goto out;
3599
3600 val = hba->ee_ctrl_mask & ~mask;
3601 val &= 0xFFFF;
3602 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3603 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3604 if (!err)
3605 hba->ee_ctrl_mask &= ~mask;
3606out:
3607 return err;
3608}
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
3621{
3622 int err = 0;
3623 u32 val;
3624
3625 if (hba->ee_ctrl_mask & mask)
3626 goto out;
3627
3628 val = hba->ee_ctrl_mask | mask;
3629 val &= 0xFFFF;
3630 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3631 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3632 if (!err)
3633 hba->ee_ctrl_mask |= mask;
3634out:
3635 return err;
3636}
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
3650{
3651 int err = 0;
3652
3653 if (hba->auto_bkops_enabled)
3654 goto out;
3655
3656 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
3657 QUERY_FLAG_IDN_BKOPS_EN, NULL);
3658 if (err) {
3659 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
3660 __func__, err);
3661 goto out;
3662 }
3663
3664 hba->auto_bkops_enabled = true;
3665
3666
3667 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3668 if (err)
3669 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
3670 __func__, err);
3671out:
3672 return err;
3673}
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
3688{
3689 int err = 0;
3690
3691 if (!hba->auto_bkops_enabled)
3692 goto out;
3693
3694
3695
3696
3697
3698 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
3699 if (err) {
3700 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
3701 __func__, err);
3702 goto out;
3703 }
3704
3705 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
3706 QUERY_FLAG_IDN_BKOPS_EN, NULL);
3707 if (err) {
3708 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
3709 __func__, err);
3710 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3711 goto out;
3712 }
3713
3714 hba->auto_bkops_enabled = false;
3715out:
3716 return err;
3717}
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
3728{
3729 hba->auto_bkops_enabled = false;
3730 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
3731 ufshcd_enable_auto_bkops(hba);
3732}
3733
3734static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
3735{
3736 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3737 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
3738}
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
3757 enum bkops_status status)
3758{
3759 int err;
3760 u32 curr_status = 0;
3761
3762 err = ufshcd_get_bkops_status(hba, &curr_status);
3763 if (err) {
3764 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
3765 __func__, err);
3766 goto out;
3767 } else if (curr_status > BKOPS_STATUS_MAX) {
3768 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
3769 __func__, curr_status);
3770 err = -EINVAL;
3771 goto out;
3772 }
3773
3774 if (curr_status >= status)
3775 err = ufshcd_enable_auto_bkops(hba);
3776 else
3777 err = ufshcd_disable_auto_bkops(hba);
3778out:
3779 return err;
3780}
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792static int ufshcd_urgent_bkops(struct ufs_hba *hba)
3793{
3794 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
3795}
3796
3797static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
3798{
3799 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3800 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
3801}
3802
3803static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
3804{
3805 int err;
3806 u32 curr_status = 0;
3807
3808 if (hba->is_urgent_bkops_lvl_checked)
3809 goto enable_auto_bkops;
3810
3811 err = ufshcd_get_bkops_status(hba, &curr_status);
3812 if (err) {
3813 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
3814 __func__, err);
3815 goto out;
3816 }
3817
3818
3819
3820
3821
3822
3823
3824 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
3825 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
3826 __func__, curr_status);
3827
3828 hba->urgent_bkops_lvl = curr_status;
3829 hba->is_urgent_bkops_lvl_checked = true;
3830 }
3831
3832enable_auto_bkops:
3833 err = ufshcd_enable_auto_bkops(hba);
3834out:
3835 if (err < 0)
3836 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
3837 __func__, err);
3838}
3839
3840
3841
3842
3843
3844
3845
3846
3847static void ufshcd_exception_event_handler(struct work_struct *work)
3848{
3849 struct ufs_hba *hba;
3850 int err;
3851 u32 status = 0;
3852 hba = container_of(work, struct ufs_hba, eeh_work);
3853
3854 pm_runtime_get_sync(hba->dev);
3855 err = ufshcd_get_ee_status(hba, &status);
3856 if (err) {
3857 dev_err(hba->dev, "%s: failed to get exception status %d\n",
3858 __func__, err);
3859 goto out;
3860 }
3861
3862 status &= hba->ee_ctrl_mask;
3863
3864 if (status & MASK_EE_URGENT_BKOPS)
3865 ufshcd_bkops_exception_event_handler(hba);
3866
3867out:
3868 pm_runtime_put_sync(hba->dev);
3869 return;
3870}
3871
3872
3873static void ufshcd_complete_requests(struct ufs_hba *hba)
3874{
3875 ufshcd_transfer_req_compl(hba);
3876 ufshcd_tmc_handler(hba);
3877}
3878
3879
3880
3881
3882
3883
3884
3885
3886static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
3887{
3888 unsigned long flags;
3889 bool err_handling = true;
3890
3891 spin_lock_irqsave(hba->host->host_lock, flags);
3892
3893
3894
3895
3896 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
3897 goto out;
3898
3899 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
3900 ((hba->saved_err & UIC_ERROR) &&
3901 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
3902 goto out;
3903
3904 if ((hba->saved_err & UIC_ERROR) &&
3905 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
3906 int err;
3907
3908
3909
3910 spin_unlock_irqrestore(hba->host->host_lock, flags);
3911 msleep(50);
3912 spin_lock_irqsave(hba->host->host_lock, flags);
3913
3914
3915
3916
3917
3918 if ((hba->saved_err & INT_FATAL_ERRORS) ||
3919 ((hba->saved_err & UIC_ERROR) &&
3920 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
3921 goto out;
3922
3923
3924
3925
3926
3927
3928
3929
3930 spin_unlock_irqrestore(hba->host->host_lock, flags);
3931 err = ufshcd_verify_dev_init(hba);
3932 spin_lock_irqsave(hba->host->host_lock, flags);
3933
3934 if (err)
3935 goto out;
3936
3937
3938 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
3939 hba->saved_err &= ~UIC_ERROR;
3940
3941 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
3942 if (!hba->saved_uic_err) {
3943 err_handling = false;
3944 goto out;
3945 }
3946 }
3947out:
3948 spin_unlock_irqrestore(hba->host->host_lock, flags);
3949 return err_handling;
3950}
3951
3952
3953
3954
3955
3956static void ufshcd_err_handler(struct work_struct *work)
3957{
3958 struct ufs_hba *hba;
3959 unsigned long flags;
3960 u32 err_xfer = 0;
3961 u32 err_tm = 0;
3962 int err = 0;
3963 int tag;
3964 bool needs_reset = false;
3965
3966 hba = container_of(work, struct ufs_hba, eh_work);
3967
3968 pm_runtime_get_sync(hba->dev);
3969 ufshcd_hold(hba, false);
3970
3971 spin_lock_irqsave(hba->host->host_lock, flags);
3972 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
3973 goto out;
3974
3975 hba->ufshcd_state = UFSHCD_STATE_RESET;
3976 ufshcd_set_eh_in_progress(hba);
3977
3978
3979 ufshcd_complete_requests(hba);
3980
3981 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
3982 bool ret;
3983
3984 spin_unlock_irqrestore(hba->host->host_lock, flags);
3985
3986 ret = ufshcd_quirk_dl_nac_errors(hba);
3987 spin_lock_irqsave(hba->host->host_lock, flags);
3988 if (!ret)
3989 goto skip_err_handling;
3990 }
3991 if ((hba->saved_err & INT_FATAL_ERRORS) ||
3992 ((hba->saved_err & UIC_ERROR) &&
3993 (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
3994 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
3995 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
3996 needs_reset = true;
3997
3998
3999
4000
4001
4002
4003 if (needs_reset)
4004 goto skip_pending_xfer_clear;
4005
4006
4007 spin_unlock_irqrestore(hba->host->host_lock, flags);
4008
4009 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
4010 if (ufshcd_clear_cmd(hba, tag)) {
4011 err_xfer = true;
4012 goto lock_skip_pending_xfer_clear;
4013 }
4014 }
4015
4016
4017 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
4018 if (ufshcd_clear_tm_cmd(hba, tag)) {
4019 err_tm = true;
4020 goto lock_skip_pending_xfer_clear;
4021 }
4022 }
4023
4024lock_skip_pending_xfer_clear:
4025 spin_lock_irqsave(hba->host->host_lock, flags);
4026
4027
4028 ufshcd_complete_requests(hba);
4029
4030 if (err_xfer || err_tm)
4031 needs_reset = true;
4032
4033skip_pending_xfer_clear:
4034
4035 if (needs_reset) {
4036 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
4037
4038
4039
4040
4041
4042
4043
4044
4045 if (hba->outstanding_reqs == max_doorbells)
4046 __ufshcd_transfer_req_compl(hba,
4047 (1UL << (hba->nutrs - 1)));
4048
4049 spin_unlock_irqrestore(hba->host->host_lock, flags);
4050 err = ufshcd_reset_and_restore(hba);
4051 spin_lock_irqsave(hba->host->host_lock, flags);
4052 if (err) {
4053 dev_err(hba->dev, "%s: reset and restore failed\n",
4054 __func__);
4055 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4056 }
4057
4058
4059
4060
4061 scsi_report_bus_reset(hba->host, 0);
4062 hba->saved_err = 0;
4063 hba->saved_uic_err = 0;
4064 }
4065
4066skip_err_handling:
4067 if (!needs_reset) {
4068 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4069 if (hba->saved_err || hba->saved_uic_err)
4070 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
4071 __func__, hba->saved_err, hba->saved_uic_err);
4072 }
4073
4074 ufshcd_clear_eh_in_progress(hba);
4075
4076out:
4077 spin_unlock_irqrestore(hba->host->host_lock, flags);
4078 scsi_unblock_requests(hba->host);
4079 ufshcd_release(hba);
4080 pm_runtime_put_sync(hba->dev);
4081}
4082
4083
4084
4085
4086
4087static void ufshcd_update_uic_error(struct ufs_hba *hba)
4088{
4089 u32 reg;
4090
4091
4092 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
4093 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
4094 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
4095 else if (hba->dev_quirks &
4096 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
4097 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
4098 hba->uic_error |=
4099 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
4100 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
4101 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
4102 }
4103
4104
4105 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
4106 if (reg)
4107 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
4108
4109 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
4110 if (reg)
4111 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
4112
4113 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
4114 if (reg)
4115 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
4116
4117 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
4118 __func__, hba->uic_error);
4119}
4120
4121
4122
4123
4124
4125static void ufshcd_check_errors(struct ufs_hba *hba)
4126{
4127 bool queue_eh_work = false;
4128
4129 if (hba->errors & INT_FATAL_ERRORS)
4130 queue_eh_work = true;
4131
4132 if (hba->errors & UIC_ERROR) {
4133 hba->uic_error = 0;
4134 ufshcd_update_uic_error(hba);
4135 if (hba->uic_error)
4136 queue_eh_work = true;
4137 }
4138
4139 if (queue_eh_work) {
4140
4141
4142
4143
4144 hba->saved_err |= hba->errors;
4145 hba->saved_uic_err |= hba->uic_error;
4146
4147
4148 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
4149
4150 scsi_block_requests(hba->host);
4151
4152 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4153 schedule_work(&hba->eh_work);
4154 }
4155 }
4156
4157
4158
4159
4160
4161
4162}
4163
4164
4165
4166
4167
4168static void ufshcd_tmc_handler(struct ufs_hba *hba)
4169{
4170 u32 tm_doorbell;
4171
4172 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
4173 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
4174 wake_up(&hba->tm_wq);
4175}
4176
4177
4178
4179
4180
4181
4182static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
4183{
4184 hba->errors = UFSHCD_ERROR_MASK & intr_status;
4185 if (hba->errors)
4186 ufshcd_check_errors(hba);
4187
4188 if (intr_status & UFSHCD_UIC_MASK)
4189 ufshcd_uic_cmd_compl(hba, intr_status);
4190
4191 if (intr_status & UTP_TASK_REQ_COMPL)
4192 ufshcd_tmc_handler(hba);
4193
4194 if (intr_status & UTP_TRANSFER_REQ_COMPL)
4195 ufshcd_transfer_req_compl(hba);
4196}
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206static irqreturn_t ufshcd_intr(int irq, void *__hba)
4207{
4208 u32 intr_status, enabled_intr_status;
4209 irqreturn_t retval = IRQ_NONE;
4210 struct ufs_hba *hba = __hba;
4211
4212 spin_lock(hba->host->host_lock);
4213 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
4214 enabled_intr_status =
4215 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4216
4217 if (intr_status)
4218 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
4219
4220 if (enabled_intr_status) {
4221 ufshcd_sl_intr(hba, enabled_intr_status);
4222 retval = IRQ_HANDLED;
4223 }
4224 spin_unlock(hba->host->host_lock);
4225 return retval;
4226}
4227
4228static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
4229{
4230 int err = 0;
4231 u32 mask = 1 << tag;
4232 unsigned long flags;
4233
4234 if (!test_bit(tag, &hba->outstanding_tasks))
4235 goto out;
4236
4237 spin_lock_irqsave(hba->host->host_lock, flags);
4238 ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
4239 spin_unlock_irqrestore(hba->host->host_lock, flags);
4240
4241
4242 err = ufshcd_wait_for_register(hba,
4243 REG_UTP_TASK_REQ_DOOR_BELL,
4244 mask, 0, 1000, 1000, true);
4245out:
4246 return err;
4247}
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
4260 u8 tm_function, u8 *tm_response)
4261{
4262 struct utp_task_req_desc *task_req_descp;
4263 struct utp_upiu_task_req *task_req_upiup;
4264 struct Scsi_Host *host;
4265 unsigned long flags;
4266 int free_slot;
4267 int err;
4268 int task_tag;
4269
4270 host = hba->host;
4271
4272
4273
4274
4275
4276
4277 wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot));
4278 ufshcd_hold(hba, false);
4279
4280 spin_lock_irqsave(host->host_lock, flags);
4281 task_req_descp = hba->utmrdl_base_addr;
4282 task_req_descp += free_slot;
4283
4284
4285 task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
4286 task_req_descp->header.dword_2 =
4287 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
4288
4289
4290 task_req_upiup =
4291 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
4292 task_tag = hba->nutrs + free_slot;
4293 task_req_upiup->header.dword_0 =
4294 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
4295 lun_id, task_tag);
4296 task_req_upiup->header.dword_1 =
4297 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
4298
4299
4300
4301
4302 task_req_upiup->input_param1 = cpu_to_be32(lun_id);
4303 task_req_upiup->input_param2 = cpu_to_be32(task_id);
4304
4305
4306 __set_bit(free_slot, &hba->outstanding_tasks);
4307
4308
4309 wmb();
4310
4311 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
4312
4313 spin_unlock_irqrestore(host->host_lock, flags);
4314
4315
4316 err = wait_event_timeout(hba->tm_wq,
4317 test_bit(free_slot, &hba->tm_condition),
4318 msecs_to_jiffies(TM_CMD_TIMEOUT));
4319 if (!err) {
4320 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
4321 __func__, tm_function);
4322 if (ufshcd_clear_tm_cmd(hba, free_slot))
4323 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
4324 __func__, free_slot);
4325 err = -ETIMEDOUT;
4326 } else {
4327 err = ufshcd_task_req_compl(hba, free_slot, tm_response);
4328 }
4329
4330 clear_bit(free_slot, &hba->tm_condition);
4331 ufshcd_put_tm_slot(hba, free_slot);
4332 wake_up(&hba->tm_tag_wq);
4333
4334 ufshcd_release(hba);
4335 return err;
4336}
4337
4338
4339
4340
4341
4342
4343
4344
4345static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
4346{
4347 struct Scsi_Host *host;
4348 struct ufs_hba *hba;
4349 unsigned int tag;
4350 u32 pos;
4351 int err;
4352 u8 resp = 0xF;
4353 struct ufshcd_lrb *lrbp;
4354 unsigned long flags;
4355
4356 host = cmd->device->host;
4357 hba = shost_priv(host);
4358 tag = cmd->request->tag;
4359
4360 lrbp = &hba->lrb[tag];
4361 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
4362 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4363 if (!err)
4364 err = resp;
4365 goto out;
4366 }
4367
4368
4369 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
4370 if (hba->lrb[pos].lun == lrbp->lun) {
4371 err = ufshcd_clear_cmd(hba, pos);
4372 if (err)
4373 break;
4374 }
4375 }
4376 spin_lock_irqsave(host->host_lock, flags);
4377 ufshcd_transfer_req_compl(hba);
4378 spin_unlock_irqrestore(host->host_lock, flags);
4379out:
4380 if (!err) {
4381 err = SUCCESS;
4382 } else {
4383 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4384 err = FAILED;
4385 }
4386 return err;
4387}
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401static int ufshcd_abort(struct scsi_cmnd *cmd)
4402{
4403 struct Scsi_Host *host;
4404 struct ufs_hba *hba;
4405 unsigned long flags;
4406 unsigned int tag;
4407 int err = 0;
4408 int poll_cnt;
4409 u8 resp = 0xF;
4410 struct ufshcd_lrb *lrbp;
4411 u32 reg;
4412
4413 host = cmd->device->host;
4414 hba = shost_priv(host);
4415 tag = cmd->request->tag;
4416 if (!ufshcd_valid_tag(hba, tag)) {
4417 dev_err(hba->dev,
4418 "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
4419 __func__, tag, cmd, cmd->request);
4420 BUG();
4421 }
4422
4423 ufshcd_hold(hba, false);
4424 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4425
4426 if (!(test_bit(tag, &hba->outstanding_reqs))) {
4427 dev_err(hba->dev,
4428 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
4429 __func__, tag, hba->outstanding_reqs, reg);
4430 goto out;
4431 }
4432
4433 if (!(reg & (1 << tag))) {
4434 dev_err(hba->dev,
4435 "%s: cmd was completed, but without a notifying intr, tag = %d",
4436 __func__, tag);
4437 }
4438
4439 lrbp = &hba->lrb[tag];
4440 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
4441 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4442 UFS_QUERY_TASK, &resp);
4443 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
4444
4445 break;
4446 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4447
4448
4449
4450
4451 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4452 if (reg & (1 << tag)) {
4453
4454 usleep_range(100, 200);
4455 continue;
4456 }
4457
4458 goto out;
4459 } else {
4460 if (!err)
4461 err = resp;
4462 goto out;
4463 }
4464 }
4465
4466 if (!poll_cnt) {
4467 err = -EBUSY;
4468 goto out;
4469 }
4470
4471 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4472 UFS_ABORT_TASK, &resp);
4473 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4474 if (!err)
4475 err = resp;
4476 goto out;
4477 }
4478
4479 err = ufshcd_clear_cmd(hba, tag);
4480 if (err)
4481 goto out;
4482
4483 scsi_dma_unmap(cmd);
4484
4485 spin_lock_irqsave(host->host_lock, flags);
4486 ufshcd_outstanding_req_clear(hba, tag);
4487 hba->lrb[tag].cmd = NULL;
4488 spin_unlock_irqrestore(host->host_lock, flags);
4489
4490 clear_bit_unlock(tag, &hba->lrb_in_use);
4491 wake_up(&hba->dev_cmd.tag_wq);
4492
4493out:
4494 if (!err) {
4495 err = SUCCESS;
4496 } else {
4497 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4498 err = FAILED;
4499 }
4500
4501
4502
4503
4504
4505 ufshcd_release(hba);
4506 return err;
4507}
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
4520{
4521 int err;
4522 unsigned long flags;
4523
4524
4525 spin_lock_irqsave(hba->host->host_lock, flags);
4526 ufshcd_hba_stop(hba, false);
4527 spin_unlock_irqrestore(hba->host->host_lock, flags);
4528
4529 err = ufshcd_hba_enable(hba);
4530 if (err)
4531 goto out;
4532
4533
4534 err = ufshcd_probe_hba(hba);
4535
4536 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
4537 err = -EIO;
4538out:
4539 if (err)
4540 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
4541
4542 return err;
4543}
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554static int ufshcd_reset_and_restore(struct ufs_hba *hba)
4555{
4556 int err = 0;
4557 unsigned long flags;
4558 int retries = MAX_HOST_RESET_RETRIES;
4559
4560 do {
4561 err = ufshcd_host_reset_and_restore(hba);
4562 } while (err && --retries);
4563
4564
4565
4566
4567
4568 spin_lock_irqsave(hba->host->host_lock, flags);
4569 ufshcd_transfer_req_compl(hba);
4570 ufshcd_tmc_handler(hba);
4571 spin_unlock_irqrestore(hba->host->host_lock, flags);
4572
4573 return err;
4574}
4575
4576
4577
4578
4579
4580
4581
4582static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
4583{
4584 int err;
4585 unsigned long flags;
4586 struct ufs_hba *hba;
4587
4588 hba = shost_priv(cmd->device->host);
4589
4590 ufshcd_hold(hba, false);
4591
4592
4593
4594
4595
4596
4597 do {
4598 spin_lock_irqsave(hba->host->host_lock, flags);
4599 if (!(work_pending(&hba->eh_work) ||
4600 hba->ufshcd_state == UFSHCD_STATE_RESET))
4601 break;
4602 spin_unlock_irqrestore(hba->host->host_lock, flags);
4603 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
4604 flush_work(&hba->eh_work);
4605 } while (1);
4606
4607 hba->ufshcd_state = UFSHCD_STATE_RESET;
4608 ufshcd_set_eh_in_progress(hba);
4609 spin_unlock_irqrestore(hba->host->host_lock, flags);
4610
4611 err = ufshcd_reset_and_restore(hba);
4612
4613 spin_lock_irqsave(hba->host->host_lock, flags);
4614 if (!err) {
4615 err = SUCCESS;
4616 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4617 } else {
4618 err = FAILED;
4619 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4620 }
4621 ufshcd_clear_eh_in_progress(hba);
4622 spin_unlock_irqrestore(hba->host->host_lock, flags);
4623
4624 ufshcd_release(hba);
4625 return err;
4626}
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
4637{
4638 int i;
4639 int curr_uA;
4640 u16 data;
4641 u16 unit;
4642
4643 for (i = start_scan; i >= 0; i--) {
4644 data = be16_to_cpu(*((u16 *)(buff + 2*i)));
4645 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
4646 ATTR_ICC_LVL_UNIT_OFFSET;
4647 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
4648 switch (unit) {
4649 case UFSHCD_NANO_AMP:
4650 curr_uA = curr_uA / 1000;
4651 break;
4652 case UFSHCD_MILI_AMP:
4653 curr_uA = curr_uA * 1000;
4654 break;
4655 case UFSHCD_AMP:
4656 curr_uA = curr_uA * 1000 * 1000;
4657 break;
4658 case UFSHCD_MICRO_AMP:
4659 default:
4660 break;
4661 }
4662 if (sup_curr_uA >= curr_uA)
4663 break;
4664 }
4665 if (i < 0) {
4666 i = 0;
4667 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
4668 }
4669
4670 return (u32)i;
4671}
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
4683 u8 *desc_buf, int len)
4684{
4685 u32 icc_level = 0;
4686
4687 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
4688 !hba->vreg_info.vccq2) {
4689 dev_err(hba->dev,
4690 "%s: Regulator capability was not set, actvIccLevel=%d",
4691 __func__, icc_level);
4692 goto out;
4693 }
4694
4695 if (hba->vreg_info.vcc)
4696 icc_level = ufshcd_get_max_icc_level(
4697 hba->vreg_info.vcc->max_uA,
4698 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
4699 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
4700
4701 if (hba->vreg_info.vccq)
4702 icc_level = ufshcd_get_max_icc_level(
4703 hba->vreg_info.vccq->max_uA,
4704 icc_level,
4705 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
4706
4707 if (hba->vreg_info.vccq2)
4708 icc_level = ufshcd_get_max_icc_level(
4709 hba->vreg_info.vccq2->max_uA,
4710 icc_level,
4711 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
4712out:
4713 return icc_level;
4714}
4715
4716static void ufshcd_init_icc_levels(struct ufs_hba *hba)
4717{
4718 int ret;
4719 int buff_len = QUERY_DESC_POWER_MAX_SIZE;
4720 u8 desc_buf[QUERY_DESC_POWER_MAX_SIZE];
4721
4722 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
4723 if (ret) {
4724 dev_err(hba->dev,
4725 "%s: Failed reading power descriptor.len = %d ret = %d",
4726 __func__, buff_len, ret);
4727 return;
4728 }
4729
4730 hba->init_prefetch_data.icc_level =
4731 ufshcd_find_max_sup_active_icc_level(hba,
4732 desc_buf, buff_len);
4733 dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
4734 __func__, hba->init_prefetch_data.icc_level);
4735
4736 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4737 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
4738 &hba->init_prefetch_data.icc_level);
4739
4740 if (ret)
4741 dev_err(hba->dev,
4742 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
4743 __func__, hba->init_prefetch_data.icc_level , ret);
4744
4745}
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
4774{
4775 int ret = 0;
4776 struct scsi_device *sdev_rpmb;
4777 struct scsi_device *sdev_boot;
4778
4779 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
4780 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
4781 if (IS_ERR(hba->sdev_ufs_device)) {
4782 ret = PTR_ERR(hba->sdev_ufs_device);
4783 hba->sdev_ufs_device = NULL;
4784 goto out;
4785 }
4786 scsi_device_put(hba->sdev_ufs_device);
4787
4788 sdev_boot = __scsi_add_device(hba->host, 0, 0,
4789 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
4790 if (IS_ERR(sdev_boot)) {
4791 ret = PTR_ERR(sdev_boot);
4792 goto remove_sdev_ufs_device;
4793 }
4794 scsi_device_put(sdev_boot);
4795
4796 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
4797 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
4798 if (IS_ERR(sdev_rpmb)) {
4799 ret = PTR_ERR(sdev_rpmb);
4800 goto remove_sdev_boot;
4801 }
4802 scsi_device_put(sdev_rpmb);
4803 goto out;
4804
4805remove_sdev_boot:
4806 scsi_remove_device(sdev_boot);
4807remove_sdev_ufs_device:
4808 scsi_remove_device(hba->sdev_ufs_device);
4809out:
4810 return ret;
4811}
4812
4813static int ufs_get_device_info(struct ufs_hba *hba,
4814 struct ufs_device_info *card_data)
4815{
4816 int err;
4817 u8 model_index;
4818 u8 str_desc_buf[QUERY_DESC_STRING_MAX_SIZE + 1] = {0};
4819 u8 desc_buf[QUERY_DESC_DEVICE_MAX_SIZE];
4820
4821 err = ufshcd_read_device_desc(hba, desc_buf,
4822 QUERY_DESC_DEVICE_MAX_SIZE);
4823 if (err) {
4824 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
4825 __func__, err);
4826 goto out;
4827 }
4828
4829
4830
4831
4832
4833 card_data->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
4834 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
4835
4836 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4837
4838 err = ufshcd_read_string_desc(hba, model_index, str_desc_buf,
4839 QUERY_DESC_STRING_MAX_SIZE, ASCII_STD);
4840 if (err) {
4841 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
4842 __func__, err);
4843 goto out;
4844 }
4845
4846 str_desc_buf[QUERY_DESC_STRING_MAX_SIZE] = '\0';
4847 strlcpy(card_data->model, (str_desc_buf + QUERY_DESC_HDR_SIZE),
4848 min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET],
4849 MAX_MODEL_LEN));
4850
4851
4852 card_data->model[MAX_MODEL_LEN] = '\0';
4853
4854out:
4855 return err;
4856}
4857
4858void ufs_advertise_fixup_device(struct ufs_hba *hba)
4859{
4860 int err;
4861 struct ufs_dev_fix *f;
4862 struct ufs_device_info card_data;
4863
4864 card_data.wmanufacturerid = 0;
4865
4866 err = ufs_get_device_info(hba, &card_data);
4867 if (err) {
4868 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
4869 __func__, err);
4870 return;
4871 }
4872
4873 for (f = ufs_fixups; f->quirk; f++) {
4874 if (((f->card.wmanufacturerid == card_data.wmanufacturerid) ||
4875 (f->card.wmanufacturerid == UFS_ANY_VENDOR)) &&
4876 (STR_PRFX_EQUAL(f->card.model, card_data.model) ||
4877 !strcmp(f->card.model, UFS_ANY_MODEL)))
4878 hba->dev_quirks |= f->quirk;
4879 }
4880}
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
4894{
4895 int ret = 0;
4896 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
4897
4898 ret = ufshcd_dme_peer_get(hba,
4899 UIC_ARG_MIB_SEL(
4900 RX_MIN_ACTIVATETIME_CAPABILITY,
4901 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
4902 &peer_rx_min_activatetime);
4903 if (ret)
4904 goto out;
4905
4906
4907 tuned_pa_tactivate =
4908 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
4909 / PA_TACTIVATE_TIME_UNIT_US);
4910 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
4911 tuned_pa_tactivate);
4912
4913out:
4914 return ret;
4915}
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
4929{
4930 int ret = 0;
4931 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
4932 u32 max_hibern8_time, tuned_pa_hibern8time;
4933
4934 ret = ufshcd_dme_get(hba,
4935 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
4936 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
4937 &local_tx_hibern8_time_cap);
4938 if (ret)
4939 goto out;
4940
4941 ret = ufshcd_dme_peer_get(hba,
4942 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
4943 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
4944 &peer_rx_hibern8_time_cap);
4945 if (ret)
4946 goto out;
4947
4948 max_hibern8_time = max(local_tx_hibern8_time_cap,
4949 peer_rx_hibern8_time_cap);
4950
4951 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
4952 / PA_HIBERN8_TIME_UNIT_US);
4953 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
4954 tuned_pa_hibern8time);
4955out:
4956 return ret;
4957}
4958
4959static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
4960{
4961 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
4962 ufshcd_tune_pa_tactivate(hba);
4963 ufshcd_tune_pa_hibern8time(hba);
4964 }
4965
4966 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
4967
4968 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
4969}
4970
4971
4972
4973
4974
4975
4976
4977static int ufshcd_probe_hba(struct ufs_hba *hba)
4978{
4979 int ret;
4980
4981 ret = ufshcd_link_startup(hba);
4982 if (ret)
4983 goto out;
4984
4985 ufshcd_init_pwr_info(hba);
4986
4987
4988 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
4989 hba->is_urgent_bkops_lvl_checked = false;
4990
4991
4992 ufshcd_set_link_active(hba);
4993
4994 ret = ufshcd_verify_dev_init(hba);
4995 if (ret)
4996 goto out;
4997
4998 ret = ufshcd_complete_dev_init(hba);
4999 if (ret)
5000 goto out;
5001
5002 ufs_advertise_fixup_device(hba);
5003 ufshcd_tune_unipro_params(hba);
5004
5005 ret = ufshcd_set_vccq_rail_unused(hba,
5006 (hba->dev_quirks & UFS_DEVICE_NO_VCCQ) ? true : false);
5007 if (ret)
5008 goto out;
5009
5010
5011 ufshcd_set_ufs_dev_active(hba);
5012 ufshcd_force_reset_auto_bkops(hba);
5013 hba->wlun_dev_clr_ua = true;
5014
5015 if (ufshcd_get_max_pwr_mode(hba)) {
5016 dev_err(hba->dev,
5017 "%s: Failed getting max supported power mode\n",
5018 __func__);
5019 } else {
5020 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
5021 if (ret)
5022 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
5023 __func__, ret);
5024 }
5025
5026
5027 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5028
5029
5030
5031
5032 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
5033 bool flag;
5034
5035
5036 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
5037 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
5038 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
5039 hba->dev_info.f_power_on_wp_en = flag;
5040
5041 if (!hba->is_init_prefetch)
5042 ufshcd_init_icc_levels(hba);
5043
5044
5045 if (ufshcd_scsi_add_wlus(hba))
5046 goto out;
5047
5048 scsi_scan_host(hba->host);
5049 pm_runtime_put_sync(hba->dev);
5050 }
5051
5052 if (!hba->is_init_prefetch)
5053 hba->is_init_prefetch = true;
5054
5055
5056 if (ufshcd_is_clkscaling_enabled(hba))
5057 devfreq_resume_device(hba->devfreq);
5058
5059out:
5060
5061
5062
5063
5064 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
5065 pm_runtime_put_sync(hba->dev);
5066 ufshcd_hba_exit(hba);
5067 }
5068
5069 return ret;
5070}
5071
5072
5073
5074
5075
5076
5077static void ufshcd_async_scan(void *data, async_cookie_t cookie)
5078{
5079 struct ufs_hba *hba = (struct ufs_hba *)data;
5080
5081 ufshcd_probe_hba(hba);
5082}
5083
5084static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
5085{
5086 unsigned long flags;
5087 struct Scsi_Host *host;
5088 struct ufs_hba *hba;
5089 int index;
5090 bool found = false;
5091
5092 if (!scmd || !scmd->device || !scmd->device->host)
5093 return BLK_EH_NOT_HANDLED;
5094
5095 host = scmd->device->host;
5096 hba = shost_priv(host);
5097 if (!hba)
5098 return BLK_EH_NOT_HANDLED;
5099
5100 spin_lock_irqsave(host->host_lock, flags);
5101
5102 for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
5103 if (hba->lrb[index].cmd == scmd) {
5104 found = true;
5105 break;
5106 }
5107 }
5108
5109 spin_unlock_irqrestore(host->host_lock, flags);
5110
5111
5112
5113
5114
5115
5116 return found ? BLK_EH_NOT_HANDLED : BLK_EH_RESET_TIMER;
5117}
5118
5119static struct scsi_host_template ufshcd_driver_template = {
5120 .module = THIS_MODULE,
5121 .name = UFSHCD,
5122 .proc_name = UFSHCD,
5123 .queuecommand = ufshcd_queuecommand,
5124 .slave_alloc = ufshcd_slave_alloc,
5125 .slave_configure = ufshcd_slave_configure,
5126 .slave_destroy = ufshcd_slave_destroy,
5127 .change_queue_depth = ufshcd_change_queue_depth,
5128 .eh_abort_handler = ufshcd_abort,
5129 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
5130 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
5131 .eh_timed_out = ufshcd_eh_timed_out,
5132 .this_id = -1,
5133 .sg_tablesize = SG_ALL,
5134 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
5135 .can_queue = UFSHCD_CAN_QUEUE,
5136 .max_host_blocked = 1,
5137 .track_queue_depth = 1,
5138};
5139
5140static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
5141 int ua)
5142{
5143 int ret;
5144
5145 if (!vreg)
5146 return 0;
5147
5148 ret = regulator_set_load(vreg->reg, ua);
5149 if (ret < 0) {
5150 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
5151 __func__, vreg->name, ua, ret);
5152 }
5153
5154 return ret;
5155}
5156
5157static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
5158 struct ufs_vreg *vreg)
5159{
5160 if (!vreg)
5161 return 0;
5162 else if (vreg->unused)
5163 return 0;
5164 else
5165 return ufshcd_config_vreg_load(hba->dev, vreg,
5166 UFS_VREG_LPM_LOAD_UA);
5167}
5168
5169static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
5170 struct ufs_vreg *vreg)
5171{
5172 if (!vreg)
5173 return 0;
5174 else if (vreg->unused)
5175 return 0;
5176 else
5177 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
5178}
5179
5180static int ufshcd_config_vreg(struct device *dev,
5181 struct ufs_vreg *vreg, bool on)
5182{
5183 int ret = 0;
5184 struct regulator *reg = vreg->reg;
5185 const char *name = vreg->name;
5186 int min_uV, uA_load;
5187
5188 BUG_ON(!vreg);
5189
5190 if (regulator_count_voltages(reg) > 0) {
5191 min_uV = on ? vreg->min_uV : 0;
5192 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
5193 if (ret) {
5194 dev_err(dev, "%s: %s set voltage failed, err=%d\n",
5195 __func__, name, ret);
5196 goto out;
5197 }
5198
5199 uA_load = on ? vreg->max_uA : 0;
5200 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
5201 if (ret)
5202 goto out;
5203 }
5204out:
5205 return ret;
5206}
5207
5208static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
5209{
5210 int ret = 0;
5211
5212 if (!vreg)
5213 goto out;
5214 else if (vreg->enabled || vreg->unused)
5215 goto out;
5216
5217 ret = ufshcd_config_vreg(dev, vreg, true);
5218 if (!ret)
5219 ret = regulator_enable(vreg->reg);
5220
5221 if (!ret)
5222 vreg->enabled = true;
5223 else
5224 dev_err(dev, "%s: %s enable failed, err=%d\n",
5225 __func__, vreg->name, ret);
5226out:
5227 return ret;
5228}
5229
5230static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
5231{
5232 int ret = 0;
5233
5234 if (!vreg)
5235 goto out;
5236 else if (!vreg->enabled || vreg->unused)
5237 goto out;
5238
5239 ret = regulator_disable(vreg->reg);
5240
5241 if (!ret) {
5242
5243 ufshcd_config_vreg(dev, vreg, false);
5244 vreg->enabled = false;
5245 } else {
5246 dev_err(dev, "%s: %s disable failed, err=%d\n",
5247 __func__, vreg->name, ret);
5248 }
5249out:
5250 return ret;
5251}
5252
5253static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
5254{
5255 int ret = 0;
5256 struct device *dev = hba->dev;
5257 struct ufs_vreg_info *info = &hba->vreg_info;
5258
5259 if (!info)
5260 goto out;
5261
5262 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
5263 if (ret)
5264 goto out;
5265
5266 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
5267 if (ret)
5268 goto out;
5269
5270 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
5271 if (ret)
5272 goto out;
5273
5274out:
5275 if (ret) {
5276 ufshcd_toggle_vreg(dev, info->vccq2, false);
5277 ufshcd_toggle_vreg(dev, info->vccq, false);
5278 ufshcd_toggle_vreg(dev, info->vcc, false);
5279 }
5280 return ret;
5281}
5282
5283static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
5284{
5285 struct ufs_vreg_info *info = &hba->vreg_info;
5286
5287 if (info)
5288 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
5289
5290 return 0;
5291}
5292
5293static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
5294{
5295 int ret = 0;
5296
5297 if (!vreg)
5298 goto out;
5299
5300 vreg->reg = devm_regulator_get(dev, vreg->name);
5301 if (IS_ERR(vreg->reg)) {
5302 ret = PTR_ERR(vreg->reg);
5303 dev_err(dev, "%s: %s get failed, err=%d\n",
5304 __func__, vreg->name, ret);
5305 }
5306out:
5307 return ret;
5308}
5309
5310static int ufshcd_init_vreg(struct ufs_hba *hba)
5311{
5312 int ret = 0;
5313 struct device *dev = hba->dev;
5314 struct ufs_vreg_info *info = &hba->vreg_info;
5315
5316 if (!info)
5317 goto out;
5318
5319 ret = ufshcd_get_vreg(dev, info->vcc);
5320 if (ret)
5321 goto out;
5322
5323 ret = ufshcd_get_vreg(dev, info->vccq);
5324 if (ret)
5325 goto out;
5326
5327 ret = ufshcd_get_vreg(dev, info->vccq2);
5328out:
5329 return ret;
5330}
5331
5332static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
5333{
5334 struct ufs_vreg_info *info = &hba->vreg_info;
5335
5336 if (info)
5337 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
5338
5339 return 0;
5340}
5341
5342static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused)
5343{
5344 int ret = 0;
5345 struct ufs_vreg_info *info = &hba->vreg_info;
5346
5347 if (!info)
5348 goto out;
5349 else if (!info->vccq)
5350 goto out;
5351
5352 if (unused) {
5353
5354 ret = ufshcd_toggle_vreg(hba->dev, info->vccq, false);
5355
5356
5357
5358
5359 if (!ret)
5360 info->vccq->unused = true;
5361 } else {
5362
5363
5364
5365
5366 info->vccq->unused = false;
5367 }
5368out:
5369 return ret;
5370}
5371
5372static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
5373 bool skip_ref_clk)
5374{
5375 int ret = 0;
5376 struct ufs_clk_info *clki;
5377 struct list_head *head = &hba->clk_list_head;
5378 unsigned long flags;
5379
5380 if (!head || list_empty(head))
5381 goto out;
5382
5383 list_for_each_entry(clki, head, list) {
5384 if (!IS_ERR_OR_NULL(clki->clk)) {
5385 if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
5386 continue;
5387
5388 if (on && !clki->enabled) {
5389 ret = clk_prepare_enable(clki->clk);
5390 if (ret) {
5391 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
5392 __func__, clki->name, ret);
5393 goto out;
5394 }
5395 } else if (!on && clki->enabled) {
5396 clk_disable_unprepare(clki->clk);
5397 }
5398 clki->enabled = on;
5399 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
5400 clki->name, on ? "en" : "dis");
5401 }
5402 }
5403
5404 ret = ufshcd_vops_setup_clocks(hba, on);
5405out:
5406 if (ret) {
5407 list_for_each_entry(clki, head, list) {
5408 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
5409 clk_disable_unprepare(clki->clk);
5410 }
5411 } else if (on) {
5412 spin_lock_irqsave(hba->host->host_lock, flags);
5413 hba->clk_gating.state = CLKS_ON;
5414 spin_unlock_irqrestore(hba->host->host_lock, flags);
5415 }
5416 return ret;
5417}
5418
5419static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
5420{
5421 return __ufshcd_setup_clocks(hba, on, false);
5422}
5423
5424static int ufshcd_init_clocks(struct ufs_hba *hba)
5425{
5426 int ret = 0;
5427 struct ufs_clk_info *clki;
5428 struct device *dev = hba->dev;
5429 struct list_head *head = &hba->clk_list_head;
5430
5431 if (!head || list_empty(head))
5432 goto out;
5433
5434 list_for_each_entry(clki, head, list) {
5435 if (!clki->name)
5436 continue;
5437
5438 clki->clk = devm_clk_get(dev, clki->name);
5439 if (IS_ERR(clki->clk)) {
5440 ret = PTR_ERR(clki->clk);
5441 dev_err(dev, "%s: %s clk get failed, %d\n",
5442 __func__, clki->name, ret);
5443 goto out;
5444 }
5445
5446 if (clki->max_freq) {
5447 ret = clk_set_rate(clki->clk, clki->max_freq);
5448 if (ret) {
5449 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
5450 __func__, clki->name,
5451 clki->max_freq, ret);
5452 goto out;
5453 }
5454 clki->curr_freq = clki->max_freq;
5455 }
5456 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
5457 clki->name, clk_get_rate(clki->clk));
5458 }
5459out:
5460 return ret;
5461}
5462
5463static int ufshcd_variant_hba_init(struct ufs_hba *hba)
5464{
5465 int err = 0;
5466
5467 if (!hba->vops)
5468 goto out;
5469
5470 err = ufshcd_vops_init(hba);
5471 if (err)
5472 goto out;
5473
5474 err = ufshcd_vops_setup_regulators(hba, true);
5475 if (err)
5476 goto out_exit;
5477
5478 goto out;
5479
5480out_exit:
5481 ufshcd_vops_exit(hba);
5482out:
5483 if (err)
5484 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
5485 __func__, ufshcd_get_var_name(hba), err);
5486 return err;
5487}
5488
5489static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
5490{
5491 if (!hba->vops)
5492 return;
5493
5494 ufshcd_vops_setup_clocks(hba, false);
5495
5496 ufshcd_vops_setup_regulators(hba, false);
5497
5498 ufshcd_vops_exit(hba);
5499}
5500
5501static int ufshcd_hba_init(struct ufs_hba *hba)
5502{
5503 int err;
5504
5505
5506
5507
5508
5509
5510
5511
5512 err = ufshcd_init_hba_vreg(hba);
5513 if (err)
5514 goto out;
5515
5516 err = ufshcd_setup_hba_vreg(hba, true);
5517 if (err)
5518 goto out;
5519
5520 err = ufshcd_init_clocks(hba);
5521 if (err)
5522 goto out_disable_hba_vreg;
5523
5524 err = ufshcd_setup_clocks(hba, true);
5525 if (err)
5526 goto out_disable_hba_vreg;
5527
5528 err = ufshcd_init_vreg(hba);
5529 if (err)
5530 goto out_disable_clks;
5531
5532 err = ufshcd_setup_vreg(hba, true);
5533 if (err)
5534 goto out_disable_clks;
5535
5536 err = ufshcd_variant_hba_init(hba);
5537 if (err)
5538 goto out_disable_vreg;
5539
5540 hba->is_powered = true;
5541 goto out;
5542
5543out_disable_vreg:
5544 ufshcd_setup_vreg(hba, false);
5545out_disable_clks:
5546 ufshcd_setup_clocks(hba, false);
5547out_disable_hba_vreg:
5548 ufshcd_setup_hba_vreg(hba, false);
5549out:
5550 return err;
5551}
5552
5553static void ufshcd_hba_exit(struct ufs_hba *hba)
5554{
5555 if (hba->is_powered) {
5556 ufshcd_variant_hba_exit(hba);
5557 ufshcd_setup_vreg(hba, false);
5558 ufshcd_setup_clocks(hba, false);
5559 ufshcd_setup_hba_vreg(hba, false);
5560 hba->is_powered = false;
5561 }
5562}
5563
5564static int
5565ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
5566{
5567 unsigned char cmd[6] = {REQUEST_SENSE,
5568 0,
5569 0,
5570 0,
5571 SCSI_SENSE_BUFFERSIZE,
5572 0};
5573 char *buffer;
5574 int ret;
5575
5576 buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
5577 if (!buffer) {
5578 ret = -ENOMEM;
5579 goto out;
5580 }
5581
5582 ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer,
5583 SCSI_SENSE_BUFFERSIZE, NULL,
5584 msecs_to_jiffies(1000), 3, NULL, REQ_PM);
5585 if (ret)
5586 pr_err("%s: failed with err %d\n", __func__, ret);
5587
5588 kfree(buffer);
5589out:
5590 return ret;
5591}
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
5603 enum ufs_dev_pwr_mode pwr_mode)
5604{
5605 unsigned char cmd[6] = { START_STOP };
5606 struct scsi_sense_hdr sshdr;
5607 struct scsi_device *sdp;
5608 unsigned long flags;
5609 int ret;
5610
5611 spin_lock_irqsave(hba->host->host_lock, flags);
5612 sdp = hba->sdev_ufs_device;
5613 if (sdp) {
5614 ret = scsi_device_get(sdp);
5615 if (!ret && !scsi_device_online(sdp)) {
5616 ret = -ENODEV;
5617 scsi_device_put(sdp);
5618 }
5619 } else {
5620 ret = -ENODEV;
5621 }
5622 spin_unlock_irqrestore(hba->host->host_lock, flags);
5623
5624 if (ret)
5625 return ret;
5626
5627
5628
5629
5630
5631
5632
5633 hba->host->eh_noresume = 1;
5634 if (hba->wlun_dev_clr_ua) {
5635 ret = ufshcd_send_request_sense(hba, sdp);
5636 if (ret)
5637 goto out;
5638
5639 hba->wlun_dev_clr_ua = false;
5640 }
5641
5642 cmd[4] = pwr_mode << 4;
5643
5644
5645
5646
5647
5648
5649 ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
5650 START_STOP_TIMEOUT, 0, NULL, REQ_PM);
5651 if (ret) {
5652 sdev_printk(KERN_WARNING, sdp,
5653 "START_STOP failed for power mode: %d, result %x\n",
5654 pwr_mode, ret);
5655 if (driver_byte(ret) & DRIVER_SENSE)
5656 scsi_print_sense_hdr(sdp, NULL, &sshdr);
5657 }
5658
5659 if (!ret)
5660 hba->curr_dev_pwr_mode = pwr_mode;
5661out:
5662 scsi_device_put(sdp);
5663 hba->host->eh_noresume = 0;
5664 return ret;
5665}
5666
5667static int ufshcd_link_state_transition(struct ufs_hba *hba,
5668 enum uic_link_state req_link_state,
5669 int check_for_bkops)
5670{
5671 int ret = 0;
5672
5673 if (req_link_state == hba->uic_link_state)
5674 return 0;
5675
5676 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
5677 ret = ufshcd_uic_hibern8_enter(hba);
5678 if (!ret)
5679 ufshcd_set_link_hibern8(hba);
5680 else
5681 goto out;
5682 }
5683
5684
5685
5686
5687 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
5688 (!check_for_bkops || (check_for_bkops &&
5689 !hba->auto_bkops_enabled))) {
5690
5691
5692
5693
5694
5695
5696
5697 ret = ufshcd_uic_hibern8_enter(hba);
5698 if (ret)
5699 goto out;
5700
5701
5702
5703
5704 ufshcd_hba_stop(hba, true);
5705
5706
5707
5708
5709 ufshcd_set_link_off(hba);
5710 }
5711
5712out:
5713 return ret;
5714}
5715
5716static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
5717{
5718
5719
5720
5721
5722
5723
5724 if (!ufshcd_is_link_active(hba) &&
5725 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
5726 usleep_range(2000, 2100);
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5741 !hba->dev_info.is_lu_power_on_wp) {
5742 ufshcd_setup_vreg(hba, false);
5743 } else if (!ufshcd_is_ufs_dev_active(hba)) {
5744 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5745 if (!ufshcd_is_link_active(hba)) {
5746 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5747 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
5748 }
5749 }
5750}
5751
5752static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
5753{
5754 int ret = 0;
5755
5756 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5757 !hba->dev_info.is_lu_power_on_wp) {
5758 ret = ufshcd_setup_vreg(hba, true);
5759 } else if (!ufshcd_is_ufs_dev_active(hba)) {
5760 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
5761 if (!ret && !ufshcd_is_link_active(hba)) {
5762 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5763 if (ret)
5764 goto vcc_disable;
5765 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5766 if (ret)
5767 goto vccq_lpm;
5768 }
5769 }
5770 goto out;
5771
5772vccq_lpm:
5773 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5774vcc_disable:
5775 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5776out:
5777 return ret;
5778}
5779
5780static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
5781{
5782 if (ufshcd_is_link_off(hba))
5783 ufshcd_setup_hba_vreg(hba, false);
5784}
5785
5786static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
5787{
5788 if (ufshcd_is_link_off(hba))
5789 ufshcd_setup_hba_vreg(hba, true);
5790}
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
5809{
5810 int ret = 0;
5811 enum ufs_pm_level pm_lvl;
5812 enum ufs_dev_pwr_mode req_dev_pwr_mode;
5813 enum uic_link_state req_link_state;
5814
5815 hba->pm_op_in_progress = 1;
5816 if (!ufshcd_is_shutdown_pm(pm_op)) {
5817 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
5818 hba->rpm_lvl : hba->spm_lvl;
5819 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
5820 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
5821 } else {
5822 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
5823 req_link_state = UIC_LINK_OFF_STATE;
5824 }
5825
5826
5827
5828
5829
5830 ufshcd_hold(hba, false);
5831 hba->clk_gating.is_suspended = true;
5832
5833 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
5834 req_link_state == UIC_LINK_ACTIVE_STATE) {
5835 goto disable_clks;
5836 }
5837
5838 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
5839 (req_link_state == hba->uic_link_state))
5840 goto out;
5841
5842
5843 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
5844 ret = -EINVAL;
5845 goto out;
5846 }
5847
5848 if (ufshcd_is_runtime_pm(pm_op)) {
5849 if (ufshcd_can_autobkops_during_suspend(hba)) {
5850
5851
5852
5853
5854
5855 ret = ufshcd_urgent_bkops(hba);
5856 if (ret)
5857 goto enable_gating;
5858 } else {
5859
5860 ufshcd_disable_auto_bkops(hba);
5861 }
5862 }
5863
5864 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
5865 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
5866 !ufshcd_is_runtime_pm(pm_op))) {
5867
5868 ufshcd_disable_auto_bkops(hba);
5869 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
5870 if (ret)
5871 goto enable_gating;
5872 }
5873
5874 ret = ufshcd_link_state_transition(hba, req_link_state, 1);
5875 if (ret)
5876 goto set_dev_active;
5877
5878 ufshcd_vreg_set_lpm(hba);
5879
5880disable_clks:
5881
5882
5883
5884
5885
5886 if (ufshcd_is_clkscaling_enabled(hba)) {
5887 devfreq_suspend_device(hba->devfreq);
5888 hba->clk_scaling.window_start_t = 0;
5889 }
5890
5891
5892
5893
5894
5895 ret = ufshcd_vops_suspend(hba, pm_op);
5896 if (ret)
5897 goto set_link_active;
5898
5899 ret = ufshcd_vops_setup_clocks(hba, false);
5900 if (ret)
5901 goto vops_resume;
5902
5903 if (!ufshcd_is_link_active(hba))
5904 ufshcd_setup_clocks(hba, false);
5905 else
5906
5907 __ufshcd_setup_clocks(hba, false, true);
5908
5909 hba->clk_gating.state = CLKS_OFF;
5910
5911
5912
5913
5914 ufshcd_disable_irq(hba);
5915
5916 ufshcd_hba_vreg_set_lpm(hba);
5917 goto out;
5918
5919vops_resume:
5920 ufshcd_vops_resume(hba, pm_op);
5921set_link_active:
5922 ufshcd_vreg_set_hpm(hba);
5923 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
5924 ufshcd_set_link_active(hba);
5925 else if (ufshcd_is_link_off(hba))
5926 ufshcd_host_reset_and_restore(hba);
5927set_dev_active:
5928 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
5929 ufshcd_disable_auto_bkops(hba);
5930enable_gating:
5931 hba->clk_gating.is_suspended = false;
5932 ufshcd_release(hba);
5933out:
5934 hba->pm_op_in_progress = 0;
5935 return ret;
5936}
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
5949{
5950 int ret;
5951 enum uic_link_state old_link_state;
5952
5953 hba->pm_op_in_progress = 1;
5954 old_link_state = hba->uic_link_state;
5955
5956 ufshcd_hba_vreg_set_hpm(hba);
5957
5958 ret = ufshcd_setup_clocks(hba, true);
5959 if (ret)
5960 goto out;
5961
5962
5963 ret = ufshcd_enable_irq(hba);
5964 if (ret)
5965 goto disable_irq_and_vops_clks;
5966
5967 ret = ufshcd_vreg_set_hpm(hba);
5968 if (ret)
5969 goto disable_irq_and_vops_clks;
5970
5971
5972
5973
5974
5975
5976 ret = ufshcd_vops_resume(hba, pm_op);
5977 if (ret)
5978 goto disable_vreg;
5979
5980 if (ufshcd_is_link_hibern8(hba)) {
5981 ret = ufshcd_uic_hibern8_exit(hba);
5982 if (!ret)
5983 ufshcd_set_link_active(hba);
5984 else
5985 goto vendor_suspend;
5986 } else if (ufshcd_is_link_off(hba)) {
5987 ret = ufshcd_host_reset_and_restore(hba);
5988
5989
5990
5991
5992 if (ret || !ufshcd_is_link_active(hba))
5993 goto vendor_suspend;
5994 }
5995
5996 if (!ufshcd_is_ufs_dev_active(hba)) {
5997 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
5998 if (ret)
5999 goto set_old_link_state;
6000 }
6001
6002
6003
6004
6005
6006 ufshcd_urgent_bkops(hba);
6007 hba->clk_gating.is_suspended = false;
6008
6009 if (ufshcd_is_clkscaling_enabled(hba))
6010 devfreq_resume_device(hba->devfreq);
6011
6012
6013 ufshcd_release(hba);
6014 goto out;
6015
6016set_old_link_state:
6017 ufshcd_link_state_transition(hba, old_link_state, 0);
6018vendor_suspend:
6019 ufshcd_vops_suspend(hba, pm_op);
6020disable_vreg:
6021 ufshcd_vreg_set_lpm(hba);
6022disable_irq_and_vops_clks:
6023 ufshcd_disable_irq(hba);
6024 ufshcd_setup_clocks(hba, false);
6025out:
6026 hba->pm_op_in_progress = 0;
6027 return ret;
6028}
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039int ufshcd_system_suspend(struct ufs_hba *hba)
6040{
6041 int ret = 0;
6042
6043 if (!hba || !hba->is_powered)
6044 return 0;
6045
6046 if (pm_runtime_suspended(hba->dev)) {
6047 if (hba->rpm_lvl == hba->spm_lvl)
6048
6049
6050
6051
6052 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
6053 hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled)
6054 goto out;
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064 ret = ufshcd_runtime_resume(hba);
6065 if (ret)
6066 goto out;
6067 }
6068
6069 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
6070out:
6071 if (!ret)
6072 hba->is_sys_suspended = true;
6073 return ret;
6074}
6075EXPORT_SYMBOL(ufshcd_system_suspend);
6076
6077
6078
6079
6080
6081
6082
6083
6084int ufshcd_system_resume(struct ufs_hba *hba)
6085{
6086 if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev))
6087
6088
6089
6090
6091 return 0;
6092
6093 return ufshcd_resume(hba, UFS_SYSTEM_PM);
6094}
6095EXPORT_SYMBOL(ufshcd_system_resume);
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105int ufshcd_runtime_suspend(struct ufs_hba *hba)
6106{
6107 if (!hba || !hba->is_powered)
6108 return 0;
6109
6110 return ufshcd_suspend(hba, UFS_RUNTIME_PM);
6111}
6112EXPORT_SYMBOL(ufshcd_runtime_suspend);
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135int ufshcd_runtime_resume(struct ufs_hba *hba)
6136{
6137 if (!hba || !hba->is_powered)
6138 return 0;
6139 else
6140 return ufshcd_resume(hba, UFS_RUNTIME_PM);
6141}
6142EXPORT_SYMBOL(ufshcd_runtime_resume);
6143
6144int ufshcd_runtime_idle(struct ufs_hba *hba)
6145{
6146 return 0;
6147}
6148EXPORT_SYMBOL(ufshcd_runtime_idle);
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158int ufshcd_shutdown(struct ufs_hba *hba)
6159{
6160 int ret = 0;
6161
6162 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
6163 goto out;
6164
6165 if (pm_runtime_suspended(hba->dev)) {
6166 ret = ufshcd_runtime_resume(hba);
6167 if (ret)
6168 goto out;
6169 }
6170
6171 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
6172out:
6173 if (ret)
6174 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
6175
6176 return 0;
6177}
6178EXPORT_SYMBOL(ufshcd_shutdown);
6179
6180
6181
6182
6183
6184
6185void ufshcd_remove(struct ufs_hba *hba)
6186{
6187 scsi_remove_host(hba->host);
6188
6189 ufshcd_disable_intr(hba, hba->intr_mask);
6190 ufshcd_hba_stop(hba, true);
6191
6192 scsi_host_put(hba->host);
6193
6194 ufshcd_exit_clk_gating(hba);
6195 if (ufshcd_is_clkscaling_enabled(hba))
6196 devfreq_remove_device(hba->devfreq);
6197 ufshcd_hba_exit(hba);
6198}
6199EXPORT_SYMBOL_GPL(ufshcd_remove);
6200
6201
6202
6203
6204
6205void ufshcd_dealloc_host(struct ufs_hba *hba)
6206{
6207 scsi_host_put(hba->host);
6208}
6209EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
6210
6211
6212
6213
6214
6215
6216
6217
6218static int ufshcd_set_dma_mask(struct ufs_hba *hba)
6219{
6220 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
6221 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
6222 return 0;
6223 }
6224 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
6225}
6226
6227
6228
6229
6230
6231
6232
6233int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
6234{
6235 struct Scsi_Host *host;
6236 struct ufs_hba *hba;
6237 int err = 0;
6238
6239 if (!dev) {
6240 dev_err(dev,
6241 "Invalid memory reference for dev is NULL\n");
6242 err = -ENODEV;
6243 goto out_error;
6244 }
6245
6246 host = scsi_host_alloc(&ufshcd_driver_template,
6247 sizeof(struct ufs_hba));
6248 if (!host) {
6249 dev_err(dev, "scsi_host_alloc failed\n");
6250 err = -ENOMEM;
6251 goto out_error;
6252 }
6253 hba = shost_priv(host);
6254 hba->host = host;
6255 hba->dev = dev;
6256 *hba_handle = hba;
6257
6258out_error:
6259 return err;
6260}
6261EXPORT_SYMBOL(ufshcd_alloc_host);
6262
6263static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
6264{
6265 int ret = 0;
6266 struct ufs_clk_info *clki;
6267 struct list_head *head = &hba->clk_list_head;
6268
6269 if (!head || list_empty(head))
6270 goto out;
6271
6272 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
6273 if (ret)
6274 return ret;
6275
6276 list_for_each_entry(clki, head, list) {
6277 if (!IS_ERR_OR_NULL(clki->clk)) {
6278 if (scale_up && clki->max_freq) {
6279 if (clki->curr_freq == clki->max_freq)
6280 continue;
6281 ret = clk_set_rate(clki->clk, clki->max_freq);
6282 if (ret) {
6283 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
6284 __func__, clki->name,
6285 clki->max_freq, ret);
6286 break;
6287 }
6288 clki->curr_freq = clki->max_freq;
6289
6290 } else if (!scale_up && clki->min_freq) {
6291 if (clki->curr_freq == clki->min_freq)
6292 continue;
6293 ret = clk_set_rate(clki->clk, clki->min_freq);
6294 if (ret) {
6295 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
6296 __func__, clki->name,
6297 clki->min_freq, ret);
6298 break;
6299 }
6300 clki->curr_freq = clki->min_freq;
6301 }
6302 }
6303 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
6304 clki->name, clk_get_rate(clki->clk));
6305 }
6306
6307 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
6308
6309out:
6310 return ret;
6311}
6312
6313static int ufshcd_devfreq_target(struct device *dev,
6314 unsigned long *freq, u32 flags)
6315{
6316 int err = 0;
6317 struct ufs_hba *hba = dev_get_drvdata(dev);
6318
6319 if (!ufshcd_is_clkscaling_enabled(hba))
6320 return -EINVAL;
6321
6322 if (*freq == UINT_MAX)
6323 err = ufshcd_scale_clks(hba, true);
6324 else if (*freq == 0)
6325 err = ufshcd_scale_clks(hba, false);
6326
6327 return err;
6328}
6329
6330static int ufshcd_devfreq_get_dev_status(struct device *dev,
6331 struct devfreq_dev_status *stat)
6332{
6333 struct ufs_hba *hba = dev_get_drvdata(dev);
6334 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
6335 unsigned long flags;
6336
6337 if (!ufshcd_is_clkscaling_enabled(hba))
6338 return -EINVAL;
6339
6340 memset(stat, 0, sizeof(*stat));
6341
6342 spin_lock_irqsave(hba->host->host_lock, flags);
6343 if (!scaling->window_start_t)
6344 goto start_window;
6345
6346 if (scaling->is_busy_started)
6347 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
6348 scaling->busy_start_t));
6349
6350 stat->total_time = jiffies_to_usecs((long)jiffies -
6351 (long)scaling->window_start_t);
6352 stat->busy_time = scaling->tot_busy_t;
6353start_window:
6354 scaling->window_start_t = jiffies;
6355 scaling->tot_busy_t = 0;
6356
6357 if (hba->outstanding_reqs) {
6358 scaling->busy_start_t = ktime_get();
6359 scaling->is_busy_started = true;
6360 } else {
6361 scaling->busy_start_t = ktime_set(0, 0);
6362 scaling->is_busy_started = false;
6363 }
6364 spin_unlock_irqrestore(hba->host->host_lock, flags);
6365 return 0;
6366}
6367
6368static struct devfreq_dev_profile ufs_devfreq_profile = {
6369 .polling_ms = 100,
6370 .target = ufshcd_devfreq_target,
6371 .get_dev_status = ufshcd_devfreq_get_dev_status,
6372};
6373
6374
6375
6376
6377
6378
6379
6380
6381int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
6382{
6383 int err;
6384 struct Scsi_Host *host = hba->host;
6385 struct device *dev = hba->dev;
6386
6387 if (!mmio_base) {
6388 dev_err(hba->dev,
6389 "Invalid memory reference for mmio_base is NULL\n");
6390 err = -ENODEV;
6391 goto out_error;
6392 }
6393
6394 hba->mmio_base = mmio_base;
6395 hba->irq = irq;
6396
6397 err = ufshcd_hba_init(hba);
6398 if (err)
6399 goto out_error;
6400
6401
6402 ufshcd_hba_capabilities(hba);
6403
6404
6405 hba->ufs_version = ufshcd_get_ufs_version(hba);
6406
6407
6408 hba->intr_mask = ufshcd_get_intr_mask(hba);
6409
6410 err = ufshcd_set_dma_mask(hba);
6411 if (err) {
6412 dev_err(hba->dev, "set dma mask failed\n");
6413 goto out_disable;
6414 }
6415
6416
6417 err = ufshcd_memory_alloc(hba);
6418 if (err) {
6419 dev_err(hba->dev, "Memory allocation failed\n");
6420 goto out_disable;
6421 }
6422
6423
6424 ufshcd_host_memory_configure(hba);
6425
6426 host->can_queue = hba->nutrs;
6427 host->cmd_per_lun = hba->nutrs;
6428 host->max_id = UFSHCD_MAX_ID;
6429 host->max_lun = UFS_MAX_LUNS;
6430 host->max_channel = UFSHCD_MAX_CHANNEL;
6431 host->unique_id = host->host_no;
6432 host->max_cmd_len = MAX_CDB_SIZE;
6433
6434 hba->max_pwr_info.is_valid = false;
6435
6436
6437 init_waitqueue_head(&hba->tm_wq);
6438 init_waitqueue_head(&hba->tm_tag_wq);
6439
6440
6441 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
6442 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
6443
6444
6445 mutex_init(&hba->uic_cmd_mutex);
6446
6447
6448 mutex_init(&hba->dev_cmd.lock);
6449
6450
6451 init_waitqueue_head(&hba->dev_cmd.tag_wq);
6452
6453 ufshcd_init_clk_gating(hba);
6454
6455
6456
6457
6458
6459
6460 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
6461 REG_INTERRUPT_STATUS);
6462 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
6463
6464
6465
6466
6467 mb();
6468
6469
6470 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
6471 if (err) {
6472 dev_err(hba->dev, "request irq failed\n");
6473 goto exit_gating;
6474 } else {
6475 hba->is_irq_enabled = true;
6476 }
6477
6478 err = scsi_add_host(host, hba->dev);
6479 if (err) {
6480 dev_err(hba->dev, "scsi_add_host failed\n");
6481 goto exit_gating;
6482 }
6483
6484
6485 err = ufshcd_hba_enable(hba);
6486 if (err) {
6487 dev_err(hba->dev, "Host controller enable failed\n");
6488 goto out_remove_scsi_host;
6489 }
6490
6491 if (ufshcd_is_clkscaling_enabled(hba)) {
6492 hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile,
6493 "simple_ondemand", NULL);
6494 if (IS_ERR(hba->devfreq)) {
6495 dev_err(hba->dev, "Unable to register with devfreq %ld\n",
6496 PTR_ERR(hba->devfreq));
6497 goto out_remove_scsi_host;
6498 }
6499
6500 devfreq_suspend_device(hba->devfreq);
6501 hba->clk_scaling.window_start_t = 0;
6502 }
6503
6504
6505 pm_runtime_get_sync(dev);
6506
6507
6508
6509
6510
6511 ufshcd_set_ufs_dev_poweroff(hba);
6512
6513 async_schedule(ufshcd_async_scan, hba);
6514
6515 return 0;
6516
6517out_remove_scsi_host:
6518 scsi_remove_host(hba->host);
6519exit_gating:
6520 ufshcd_exit_clk_gating(hba);
6521out_disable:
6522 hba->is_irq_enabled = false;
6523 scsi_host_put(host);
6524 ufshcd_hba_exit(hba);
6525out_error:
6526 return err;
6527}
6528EXPORT_SYMBOL_GPL(ufshcd_init);
6529
6530MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
6531MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
6532MODULE_DESCRIPTION("Generic UFS host controller driver Core");
6533MODULE_LICENSE("GPL");
6534MODULE_VERSION(UFSHCD_DRIVER_VERSION);
6535