1
2
3
4
5
6
7
8
9
10
11
12#ifndef _UFSHCD_H
13#define _UFSHCD_H
14
15#include <linux/module.h>
16#include <linux/kernel.h>
17#include <linux/init.h>
18#include <linux/interrupt.h>
19#include <linux/io.h>
20#include <linux/delay.h>
21#include <linux/slab.h>
22#include <linux/spinlock.h>
23#include <linux/rwsem.h>
24#include <linux/workqueue.h>
25#include <linux/errno.h>
26#include <linux/types.h>
27#include <linux/wait.h>
28#include <linux/bitops.h>
29#include <linux/pm_runtime.h>
30#include <linux/clk.h>
31#include <linux/completion.h>
32#include <linux/regulator/consumer.h>
33#include <linux/bitfield.h>
34#include <linux/devfreq.h>
35#include <linux/keyslot-manager.h>
36#include "unipro.h"
37
38#include <asm/irq.h>
39#include <asm/byteorder.h>
40#include <scsi/scsi.h>
41#include <scsi/scsi_cmnd.h>
42#include <scsi/scsi_host.h>
43#include <scsi/scsi_tcq.h>
44#include <scsi/scsi_dbg.h>
45#include <scsi/scsi_eh.h>
46
47#include "ufs.h"
48#include "ufs_quirks.h"
49#include "ufshci.h"
50
51#define UFSHCD "ufshcd"
52#define UFSHCD_DRIVER_VERSION "0.2"
53
54struct ufs_hba;
55
56enum dev_cmd_type {
57 DEV_CMD_TYPE_NOP = 0x0,
58 DEV_CMD_TYPE_QUERY = 0x1,
59};
60
61enum ufs_event_type {
62
63 UFS_EVT_PA_ERR = 0,
64 UFS_EVT_DL_ERR,
65 UFS_EVT_NL_ERR,
66 UFS_EVT_TL_ERR,
67 UFS_EVT_DME_ERR,
68
69
70 UFS_EVT_AUTO_HIBERN8_ERR,
71 UFS_EVT_FATAL_ERR,
72 UFS_EVT_LINK_STARTUP_FAIL,
73 UFS_EVT_RESUME_ERR,
74 UFS_EVT_SUSPEND_ERR,
75
76
77 UFS_EVT_DEV_RESET,
78 UFS_EVT_HOST_RESET,
79 UFS_EVT_ABORT,
80
81 UFS_EVT_CNT,
82};
83
84
85
86
87
88
89
90
91
92
93struct uic_command {
94 u32 command;
95 u32 argument1;
96 u32 argument2;
97 u32 argument3;
98 int cmd_active;
99 struct completion done;
100};
101
102
103enum ufs_pm_op {
104 UFS_RUNTIME_PM,
105 UFS_SYSTEM_PM,
106 UFS_SHUTDOWN_PM,
107};
108
109#define ufshcd_is_runtime_pm(op) ((op) == UFS_RUNTIME_PM)
110#define ufshcd_is_system_pm(op) ((op) == UFS_SYSTEM_PM)
111#define ufshcd_is_shutdown_pm(op) ((op) == UFS_SHUTDOWN_PM)
112
113
114enum uic_link_state {
115 UIC_LINK_OFF_STATE = 0,
116 UIC_LINK_ACTIVE_STATE = 1,
117 UIC_LINK_HIBERN8_STATE = 2,
118 UIC_LINK_BROKEN_STATE = 3,
119};
120
121#define ufshcd_is_link_off(hba) ((hba)->uic_link_state == UIC_LINK_OFF_STATE)
122#define ufshcd_is_link_active(hba) ((hba)->uic_link_state == \
123 UIC_LINK_ACTIVE_STATE)
124#define ufshcd_is_link_hibern8(hba) ((hba)->uic_link_state == \
125 UIC_LINK_HIBERN8_STATE)
126#define ufshcd_is_link_broken(hba) ((hba)->uic_link_state == \
127 UIC_LINK_BROKEN_STATE)
128#define ufshcd_set_link_off(hba) ((hba)->uic_link_state = UIC_LINK_OFF_STATE)
129#define ufshcd_set_link_active(hba) ((hba)->uic_link_state = \
130 UIC_LINK_ACTIVE_STATE)
131#define ufshcd_set_link_hibern8(hba) ((hba)->uic_link_state = \
132 UIC_LINK_HIBERN8_STATE)
133#define ufshcd_set_link_broken(hba) ((hba)->uic_link_state = \
134 UIC_LINK_BROKEN_STATE)
135
136#define ufshcd_set_ufs_dev_active(h) \
137 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
138#define ufshcd_set_ufs_dev_sleep(h) \
139 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
140#define ufshcd_set_ufs_dev_poweroff(h) \
141 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
142#define ufshcd_set_ufs_dev_deepsleep(h) \
143 ((h)->curr_dev_pwr_mode = UFS_DEEPSLEEP_PWR_MODE)
144#define ufshcd_is_ufs_dev_active(h) \
145 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
146#define ufshcd_is_ufs_dev_sleep(h) \
147 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
148#define ufshcd_is_ufs_dev_poweroff(h) \
149 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
150#define ufshcd_is_ufs_dev_deepsleep(h) \
151 ((h)->curr_dev_pwr_mode == UFS_DEEPSLEEP_PWR_MODE)
152
153
154
155
156
157
158
159enum ufs_pm_level {
160 UFS_PM_LVL_0,
161 UFS_PM_LVL_1,
162 UFS_PM_LVL_2,
163 UFS_PM_LVL_3,
164 UFS_PM_LVL_4,
165 UFS_PM_LVL_5,
166 UFS_PM_LVL_6,
167 UFS_PM_LVL_MAX
168};
169
170struct ufs_pm_lvl_states {
171 enum ufs_dev_pwr_mode dev_state;
172 enum uic_link_state link_state;
173};
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200struct ufshcd_lrb {
201 struct utp_transfer_req_desc *utr_descriptor_ptr;
202 struct utp_upiu_req *ucd_req_ptr;
203 struct utp_upiu_rsp *ucd_rsp_ptr;
204 struct ufshcd_sg_entry *ucd_prdt_ptr;
205
206 dma_addr_t utrd_dma_addr;
207 dma_addr_t ucd_req_dma_addr;
208 dma_addr_t ucd_rsp_dma_addr;
209 dma_addr_t ucd_prdt_dma_addr;
210
211 struct scsi_cmnd *cmd;
212 u8 *sense_buffer;
213 unsigned int sense_bufflen;
214 int scsi_status;
215
216 int command_type;
217 int task_tag;
218 u8 lun;
219 bool intr_cmd;
220 ktime_t issue_time_stamp;
221 ktime_t compl_time_stamp;
222#ifdef CONFIG_SCSI_UFS_CRYPTO
223 int crypto_key_slot;
224 u64 data_unit_num;
225#endif
226
227 bool req_abort_skip;
228 bool in_use;
229};
230
231
232
233
234
235
236
237struct ufs_query {
238 struct ufs_query_req request;
239 u8 *descriptor;
240 struct ufs_query_res response;
241};
242
243
244
245
246
247
248
249struct ufs_dev_cmd {
250 enum dev_cmd_type type;
251 struct mutex lock;
252 struct completion *complete;
253 struct ufs_query query;
254};
255
256
257
258
259
260
261
262
263
264
265
266
267
268struct ufs_clk_info {
269 struct list_head list;
270 struct clk *clk;
271 const char *name;
272 u32 max_freq;
273 u32 min_freq;
274 u32 curr_freq;
275 bool keep_link_active;
276 bool enabled;
277};
278
279enum ufs_notify_change_status {
280 PRE_CHANGE,
281 POST_CHANGE,
282};
283
284struct ufs_pa_layer_attr {
285 u32 gear_rx;
286 u32 gear_tx;
287 u32 lane_rx;
288 u32 lane_tx;
289 u32 pwr_rx;
290 u32 pwr_tx;
291 u32 hs_rate;
292};
293
294struct ufs_pwr_mode_info {
295 bool is_valid;
296 struct ufs_pa_layer_attr info;
297};
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328struct ufs_hba_variant_ops {
329 const char *name;
330 int (*init)(struct ufs_hba *);
331 void (*exit)(struct ufs_hba *);
332 u32 (*get_ufs_hci_version)(struct ufs_hba *);
333 int (*clk_scale_notify)(struct ufs_hba *, bool,
334 enum ufs_notify_change_status);
335 int (*setup_clocks)(struct ufs_hba *, bool,
336 enum ufs_notify_change_status);
337 int (*hce_enable_notify)(struct ufs_hba *,
338 enum ufs_notify_change_status);
339 int (*link_startup_notify)(struct ufs_hba *,
340 enum ufs_notify_change_status);
341 int (*pwr_change_notify)(struct ufs_hba *,
342 enum ufs_notify_change_status status,
343 struct ufs_pa_layer_attr *,
344 struct ufs_pa_layer_attr *);
345 void (*setup_xfer_req)(struct ufs_hba *, int, bool);
346 void (*setup_task_mgmt)(struct ufs_hba *, int, u8);
347 void (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme,
348 enum ufs_notify_change_status);
349 int (*apply_dev_quirks)(struct ufs_hba *hba);
350 void (*fixup_dev_quirks)(struct ufs_hba *hba);
351 int (*suspend)(struct ufs_hba *, enum ufs_pm_op);
352 int (*resume)(struct ufs_hba *, enum ufs_pm_op);
353 void (*dbg_register_dump)(struct ufs_hba *hba);
354 int (*phy_initialization)(struct ufs_hba *);
355 int (*device_reset)(struct ufs_hba *hba);
356 void (*config_scaling_param)(struct ufs_hba *hba,
357 struct devfreq_dev_profile *profile,
358 void *data);
359 int (*program_key)(struct ufs_hba *hba,
360 const union ufs_crypto_cfg_entry *cfg, int slot);
361 void (*event_notify)(struct ufs_hba *hba,
362 enum ufs_event_type evt, void *data);
363};
364
365
366enum clk_gating_state {
367 CLKS_OFF,
368 CLKS_ON,
369 REQ_CLKS_OFF,
370 REQ_CLKS_ON,
371};
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389struct ufs_clk_gating {
390 struct delayed_work gate_work;
391 struct work_struct ungate_work;
392 enum clk_gating_state state;
393 unsigned long delay_ms;
394 bool is_suspended;
395 struct device_attribute delay_attr;
396 struct device_attribute enable_attr;
397 bool is_enabled;
398 int active_reqs;
399 struct workqueue_struct *clk_gating_workq;
400};
401
402struct ufs_saved_pwr_info {
403 struct ufs_pa_layer_attr info;
404 bool is_valid;
405};
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426struct ufs_clk_scaling {
427 int active_reqs;
428 unsigned long tot_busy_t;
429 ktime_t window_start_t;
430 ktime_t busy_start_t;
431 struct device_attribute enable_attr;
432 struct ufs_saved_pwr_info saved_pwr_info;
433 struct workqueue_struct *workq;
434 struct work_struct suspend_work;
435 struct work_struct resume_work;
436 u32 min_gear;
437 bool is_allowed;
438 bool is_busy_started;
439 bool is_suspended;
440};
441
442#define UFS_EVENT_HIST_LENGTH 8
443
444
445
446
447
448
449struct ufs_event_hist {
450 int pos;
451 u32 val[UFS_EVENT_HIST_LENGTH];
452 ktime_t tstamp[UFS_EVENT_HIST_LENGTH];
453};
454
455
456
457
458
459
460
461
462
463
464struct ufs_stats {
465 u32 last_intr_status;
466 ktime_t last_intr_ts;
467
468 u32 hibern8_exit_cnt;
469 ktime_t last_hibern8_exit_tstamp;
470 struct ufs_event_hist event[UFS_EVT_CNT];
471};
472
473enum ufshcd_quirks {
474
475 UFSHCD_QUIRK_BROKEN_INTR_AGGR = 1 << 0,
476
477
478
479
480
481 UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS = 1 << 1,
482
483
484
485
486
487
488
489
490 UFSHCD_QUIRK_BROKEN_LCC = 1 << 2,
491
492
493
494
495
496
497 UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP = 1 << 3,
498
499
500
501
502
503
504 UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE = 1 << 4,
505
506
507
508
509
510
511
512 UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION = 1 << 5,
513
514
515
516
517 UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR = 1 << 6,
518
519
520
521
522
523 UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR = 1 << 7,
524
525
526
527
528
529 UFSHCI_QUIRK_BROKEN_HCE = 1 << 8,
530
531
532
533
534
535 UFSHCD_QUIRK_PRDT_BYTE_GRAN = 1 << 9,
536
537
538
539
540
541 UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR = 1 << 10,
542
543
544
545
546
547 UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 = 1 << 11,
548
549
550
551
552 UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL = 1 << 12,
553
554};
555
556enum ufshcd_caps {
557
558 UFSHCD_CAP_CLK_GATING = 1 << 0,
559
560
561 UFSHCD_CAP_HIBERN8_WITH_CLK_GATING = 1 << 1,
562
563
564 UFSHCD_CAP_CLK_SCALING = 1 << 2,
565
566
567 UFSHCD_CAP_AUTO_BKOPS_SUSPEND = 1 << 3,
568
569
570
571
572
573
574 UFSHCD_CAP_INTR_AGGR = 1 << 4,
575
576
577
578
579
580
581
582
583 UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND = 1 << 5,
584
585
586
587
588
589
590 UFSHCD_CAP_RPM_AUTOSUSPEND = 1 << 6,
591
592
593
594
595
596
597 UFSHCD_CAP_WB_EN = 1 << 7,
598
599
600
601
602
603 UFSHCD_CAP_CRYPTO = 1 << 8,
604
605
606
607
608
609
610 UFSHCD_CAP_AGGR_POWER_COLLAPSE = 1 << 9,
611
612
613
614
615
616
617
618 UFSHCD_CAP_DEEPSLEEP = 1 << 10,
619};
620
621struct ufs_hba_variant_params {
622 struct devfreq_dev_profile devfreq_profile;
623 struct devfreq_simple_ondemand_data ondemand_data;
624 u16 hba_enable_delay_us;
625 u32 wb_flush_threshold;
626};
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687struct ufs_hba {
688 void __iomem *mmio_base;
689
690
691 struct utp_transfer_cmd_desc *ucdl_base_addr;
692 struct utp_transfer_req_desc *utrdl_base_addr;
693 struct utp_task_req_desc *utmrdl_base_addr;
694
695
696 dma_addr_t ucdl_dma_addr;
697 dma_addr_t utrdl_dma_addr;
698 dma_addr_t utmrdl_dma_addr;
699
700 struct Scsi_Host *host;
701 struct device *dev;
702 struct request_queue *cmd_queue;
703
704
705
706
707 struct scsi_device *sdev_ufs_device;
708 struct scsi_device *sdev_rpmb;
709
710 enum ufs_dev_pwr_mode curr_dev_pwr_mode;
711 enum uic_link_state uic_link_state;
712
713 enum ufs_pm_level rpm_lvl;
714
715 enum ufs_pm_level spm_lvl;
716 struct device_attribute rpm_lvl_attr;
717 struct device_attribute spm_lvl_attr;
718 int pm_op_in_progress;
719
720
721 u32 ahit;
722
723 struct ufshcd_lrb *lrb;
724
725 unsigned long outstanding_tasks;
726 unsigned long outstanding_reqs;
727
728 u32 capabilities;
729 int nutrs;
730 int nutmrs;
731 u32 ufs_version;
732 const struct ufs_hba_variant_ops *vops;
733 struct ufs_hba_variant_params *vps;
734 void *priv;
735 unsigned int irq;
736 bool is_irq_enabled;
737 enum ufs_ref_clk_freq dev_ref_clk_freq;
738
739 unsigned int quirks;
740
741
742 unsigned int dev_quirks;
743
744 struct blk_mq_tag_set tmf_tag_set;
745 struct request_queue *tmf_queue;
746
747 struct uic_command *active_uic_cmd;
748 struct mutex uic_cmd_mutex;
749 struct completion *uic_async_done;
750
751 u32 ufshcd_state;
752 u32 eh_flags;
753 u32 intr_mask;
754 u16 ee_ctrl_mask;
755 bool is_powered;
756 struct semaphore eh_sem;
757
758
759 struct workqueue_struct *eh_wq;
760 struct work_struct eh_work;
761 struct work_struct eeh_work;
762
763
764 u32 errors;
765 u32 uic_error;
766 u32 saved_err;
767 u32 saved_uic_err;
768 struct ufs_stats ufs_stats;
769 bool force_reset;
770 bool force_pmc;
771 bool silence_err_logs;
772
773
774 struct ufs_dev_cmd dev_cmd;
775 ktime_t last_dme_cmd_tstamp;
776
777
778 struct ufs_dev_info dev_info;
779 bool auto_bkops_enabled;
780 struct ufs_vreg_info vreg_info;
781 struct list_head clk_list_head;
782
783 bool wlun_dev_clr_ua;
784
785
786 int req_abort_count;
787
788
789 u32 lanes_per_direction;
790 struct ufs_pa_layer_attr pwr_info;
791 struct ufs_pwr_mode_info max_pwr_info;
792
793 struct ufs_clk_gating clk_gating;
794
795 u32 caps;
796
797 struct devfreq *devfreq;
798 struct ufs_clk_scaling clk_scaling;
799 bool is_sys_suspended;
800
801 enum bkops_status urgent_bkops_lvl;
802 bool is_urgent_bkops_lvl_checked;
803
804 struct rw_semaphore clk_scaling_lock;
805 unsigned char desc_size[QUERY_DESC_IDN_MAX];
806 atomic_t scsi_block_reqs_cnt;
807
808 struct device bsg_dev;
809 struct request_queue *bsg_queue;
810 bool wb_buf_flush_enabled;
811 bool wb_enabled;
812 struct delayed_work rpm_dev_flush_recheck_work;
813
814#ifdef CONFIG_SCSI_UFS_CRYPTO
815 union ufs_crypto_capabilities crypto_capabilities;
816 union ufs_crypto_cap_entry *crypto_cap_array;
817 u32 crypto_cfg_register;
818 struct blk_keyslot_manager ksm;
819#endif
820};
821
822
823static inline bool ufshcd_is_clkgating_allowed(struct ufs_hba *hba)
824{
825 return hba->caps & UFSHCD_CAP_CLK_GATING;
826}
827static inline bool ufshcd_can_hibern8_during_gating(struct ufs_hba *hba)
828{
829 return hba->caps & UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
830}
831static inline int ufshcd_is_clkscaling_supported(struct ufs_hba *hba)
832{
833 return hba->caps & UFSHCD_CAP_CLK_SCALING;
834}
835static inline bool ufshcd_can_autobkops_during_suspend(struct ufs_hba *hba)
836{
837 return hba->caps & UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
838}
839static inline bool ufshcd_is_rpm_autosuspend_allowed(struct ufs_hba *hba)
840{
841 return hba->caps & UFSHCD_CAP_RPM_AUTOSUSPEND;
842}
843
844static inline bool ufshcd_is_intr_aggr_allowed(struct ufs_hba *hba)
845{
846
847#ifndef CONFIG_SCSI_UFS_DWC
848 if ((hba->caps & UFSHCD_CAP_INTR_AGGR) &&
849 !(hba->quirks & UFSHCD_QUIRK_BROKEN_INTR_AGGR))
850 return true;
851 else
852 return false;
853#else
854return true;
855#endif
856}
857
858static inline bool ufshcd_can_aggressive_pc(struct ufs_hba *hba)
859{
860 return !!(ufshcd_is_link_hibern8(hba) &&
861 (hba->caps & UFSHCD_CAP_AGGR_POWER_COLLAPSE));
862}
863
864static inline bool ufshcd_is_auto_hibern8_supported(struct ufs_hba *hba)
865{
866 return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) &&
867 !(hba->quirks & UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8);
868}
869
870static inline bool ufshcd_is_auto_hibern8_enabled(struct ufs_hba *hba)
871{
872 return FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit) ? true : false;
873}
874
875static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba)
876{
877 return hba->caps & UFSHCD_CAP_WB_EN;
878}
879
880#define ufshcd_writel(hba, val, reg) \
881 writel((val), (hba)->mmio_base + (reg))
882#define ufshcd_readl(hba, reg) \
883 readl((hba)->mmio_base + (reg))
884
885
886
887
888
889
890
891
892static inline void ufshcd_rmwl(struct ufs_hba *hba, u32 mask, u32 val, u32 reg)
893{
894 u32 tmp;
895
896 tmp = ufshcd_readl(hba, reg);
897 tmp &= ~mask;
898 tmp |= (val & mask);
899 ufshcd_writel(hba, tmp, reg);
900}
901
902int ufshcd_alloc_host(struct device *, struct ufs_hba **);
903void ufshcd_dealloc_host(struct ufs_hba *);
904int ufshcd_hba_enable(struct ufs_hba *hba);
905int ufshcd_init(struct ufs_hba * , void __iomem * , unsigned int);
906int ufshcd_link_recovery(struct ufs_hba *hba);
907int ufshcd_make_hba_operational(struct ufs_hba *hba);
908void ufshcd_remove(struct ufs_hba *);
909int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
910void ufshcd_delay_us(unsigned long us, unsigned long tolerance);
911int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
912 u32 val, unsigned long interval_us,
913 unsigned long timeout_ms);
914void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk);
915void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val);
916
917static inline void check_upiu_size(void)
918{
919 BUILD_BUG_ON(ALIGNED_UPIU_SIZE <
920 GENERAL_UPIU_REQUEST_SIZE + QUERY_DESC_MAX_SIZE);
921}
922
923
924
925
926
927
928static inline void ufshcd_set_variant(struct ufs_hba *hba, void *variant)
929{
930 BUG_ON(!hba);
931 hba->priv = variant;
932}
933
934
935
936
937
938static inline void *ufshcd_get_variant(struct ufs_hba *hba)
939{
940 BUG_ON(!hba);
941 return hba->priv;
942}
943static inline bool ufshcd_keep_autobkops_enabled_except_suspend(
944 struct ufs_hba *hba)
945{
946 return hba->caps & UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND;
947}
948
949static inline u8 ufshcd_wb_get_query_index(struct ufs_hba *hba)
950{
951 if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_LU_DEDICATED)
952 return hba->dev_info.wb_dedicated_lu;
953 return 0;
954}
955
956extern int ufshcd_runtime_suspend(struct ufs_hba *hba);
957extern int ufshcd_runtime_resume(struct ufs_hba *hba);
958extern int ufshcd_runtime_idle(struct ufs_hba *hba);
959extern int ufshcd_system_suspend(struct ufs_hba *hba);
960extern int ufshcd_system_resume(struct ufs_hba *hba);
961extern int ufshcd_shutdown(struct ufs_hba *hba);
962extern int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
963 int agreed_gear,
964 int adapt_val);
965extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
966 u8 attr_set, u32 mib_val, u8 peer);
967extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
968 u32 *mib_val, u8 peer);
969extern int ufshcd_config_pwr_mode(struct ufs_hba *hba,
970 struct ufs_pa_layer_attr *desired_pwr_mode);
971
972
973#define DME_LOCAL 0
974#define DME_PEER 1
975#define ATTR_SET_NOR 0
976#define ATTR_SET_ST 1
977
978static inline int ufshcd_dme_set(struct ufs_hba *hba, u32 attr_sel,
979 u32 mib_val)
980{
981 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
982 mib_val, DME_LOCAL);
983}
984
985static inline int ufshcd_dme_st_set(struct ufs_hba *hba, u32 attr_sel,
986 u32 mib_val)
987{
988 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
989 mib_val, DME_LOCAL);
990}
991
992static inline int ufshcd_dme_peer_set(struct ufs_hba *hba, u32 attr_sel,
993 u32 mib_val)
994{
995 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
996 mib_val, DME_PEER);
997}
998
999static inline int ufshcd_dme_peer_st_set(struct ufs_hba *hba, u32 attr_sel,
1000 u32 mib_val)
1001{
1002 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
1003 mib_val, DME_PEER);
1004}
1005
1006static inline int ufshcd_dme_get(struct ufs_hba *hba,
1007 u32 attr_sel, u32 *mib_val)
1008{
1009 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_LOCAL);
1010}
1011
1012static inline int ufshcd_dme_peer_get(struct ufs_hba *hba,
1013 u32 attr_sel, u32 *mib_val)
1014{
1015 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_PEER);
1016}
1017
1018static inline bool ufshcd_is_hs_mode(struct ufs_pa_layer_attr *pwr_info)
1019{
1020 return (pwr_info->pwr_rx == FAST_MODE ||
1021 pwr_info->pwr_rx == FASTAUTO_MODE) &&
1022 (pwr_info->pwr_tx == FAST_MODE ||
1023 pwr_info->pwr_tx == FASTAUTO_MODE);
1024}
1025
1026static inline int ufshcd_disable_host_tx_lcc(struct ufs_hba *hba)
1027{
1028 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0);
1029}
1030
1031
1032int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
1033 enum query_opcode opcode,
1034 enum desc_idn idn, u8 index,
1035 u8 selector,
1036 u8 *desc_buf, int *buf_len);
1037int ufshcd_read_desc_param(struct ufs_hba *hba,
1038 enum desc_idn desc_id,
1039 int desc_index,
1040 u8 param_offset,
1041 u8 *param_read_buf,
1042 u8 param_size);
1043int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1044 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val);
1045int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1046 enum flag_idn idn, u8 index, bool *flag_res);
1047
1048void ufshcd_auto_hibern8_enable(struct ufs_hba *hba);
1049void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
1050void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups);
1051#define SD_ASCII_STD true
1052#define SD_RAW false
1053int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
1054 u8 **buf, bool ascii);
1055
1056int ufshcd_hold(struct ufs_hba *hba, bool async);
1057void ufshcd_release(struct ufs_hba *hba);
1058
1059void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
1060 int *desc_length);
1061
1062u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba);
1063
1064int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
1065
1066int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
1067 struct utp_upiu_req *req_upiu,
1068 struct utp_upiu_req *rsp_upiu,
1069 int msgcode,
1070 u8 *desc_buff, int *buff_len,
1071 enum query_opcode desc_op);
1072
1073
1074static inline const char *ufshcd_get_var_name(struct ufs_hba *hba)
1075{
1076 if (hba->vops)
1077 return hba->vops->name;
1078 return "";
1079}
1080
1081static inline int ufshcd_vops_init(struct ufs_hba *hba)
1082{
1083 if (hba->vops && hba->vops->init)
1084 return hba->vops->init(hba);
1085
1086 return 0;
1087}
1088
1089static inline void ufshcd_vops_exit(struct ufs_hba *hba)
1090{
1091 if (hba->vops && hba->vops->exit)
1092 return hba->vops->exit(hba);
1093}
1094
1095static inline u32 ufshcd_vops_get_ufs_hci_version(struct ufs_hba *hba)
1096{
1097 if (hba->vops && hba->vops->get_ufs_hci_version)
1098 return hba->vops->get_ufs_hci_version(hba);
1099
1100 return ufshcd_readl(hba, REG_UFS_VERSION);
1101}
1102
1103static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba,
1104 bool up, enum ufs_notify_change_status status)
1105{
1106 if (hba->vops && hba->vops->clk_scale_notify)
1107 return hba->vops->clk_scale_notify(hba, up, status);
1108 return 0;
1109}
1110
1111static inline void ufshcd_vops_event_notify(struct ufs_hba *hba,
1112 enum ufs_event_type evt,
1113 void *data)
1114{
1115 if (hba->vops && hba->vops->event_notify)
1116 hba->vops->event_notify(hba, evt, data);
1117}
1118
1119static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on,
1120 enum ufs_notify_change_status status)
1121{
1122 if (hba->vops && hba->vops->setup_clocks)
1123 return hba->vops->setup_clocks(hba, on, status);
1124 return 0;
1125}
1126
1127static inline int ufshcd_vops_hce_enable_notify(struct ufs_hba *hba,
1128 bool status)
1129{
1130 if (hba->vops && hba->vops->hce_enable_notify)
1131 return hba->vops->hce_enable_notify(hba, status);
1132
1133 return 0;
1134}
1135static inline int ufshcd_vops_link_startup_notify(struct ufs_hba *hba,
1136 bool status)
1137{
1138 if (hba->vops && hba->vops->link_startup_notify)
1139 return hba->vops->link_startup_notify(hba, status);
1140
1141 return 0;
1142}
1143
1144static inline int ufshcd_vops_phy_initialization(struct ufs_hba *hba)
1145{
1146 if (hba->vops && hba->vops->phy_initialization)
1147 return hba->vops->phy_initialization(hba);
1148
1149 return 0;
1150}
1151
1152static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba,
1153 bool status,
1154 struct ufs_pa_layer_attr *dev_max_params,
1155 struct ufs_pa_layer_attr *dev_req_params)
1156{
1157 if (hba->vops && hba->vops->pwr_change_notify)
1158 return hba->vops->pwr_change_notify(hba, status,
1159 dev_max_params, dev_req_params);
1160
1161 return -ENOTSUPP;
1162}
1163
1164static inline void ufshcd_vops_setup_xfer_req(struct ufs_hba *hba, int tag,
1165 bool is_scsi_cmd)
1166{
1167 if (hba->vops && hba->vops->setup_xfer_req)
1168 return hba->vops->setup_xfer_req(hba, tag, is_scsi_cmd);
1169}
1170
1171static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba,
1172 int tag, u8 tm_function)
1173{
1174 if (hba->vops && hba->vops->setup_task_mgmt)
1175 return hba->vops->setup_task_mgmt(hba, tag, tm_function);
1176}
1177
1178static inline void ufshcd_vops_hibern8_notify(struct ufs_hba *hba,
1179 enum uic_cmd_dme cmd,
1180 enum ufs_notify_change_status status)
1181{
1182 if (hba->vops && hba->vops->hibern8_notify)
1183 return hba->vops->hibern8_notify(hba, cmd, status);
1184}
1185
1186static inline int ufshcd_vops_apply_dev_quirks(struct ufs_hba *hba)
1187{
1188 if (hba->vops && hba->vops->apply_dev_quirks)
1189 return hba->vops->apply_dev_quirks(hba);
1190 return 0;
1191}
1192
1193static inline void ufshcd_vops_fixup_dev_quirks(struct ufs_hba *hba)
1194{
1195 if (hba->vops && hba->vops->fixup_dev_quirks)
1196 hba->vops->fixup_dev_quirks(hba);
1197}
1198
1199static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op)
1200{
1201 if (hba->vops && hba->vops->suspend)
1202 return hba->vops->suspend(hba, op);
1203
1204 return 0;
1205}
1206
1207static inline int ufshcd_vops_resume(struct ufs_hba *hba, enum ufs_pm_op op)
1208{
1209 if (hba->vops && hba->vops->resume)
1210 return hba->vops->resume(hba, op);
1211
1212 return 0;
1213}
1214
1215static inline void ufshcd_vops_dbg_register_dump(struct ufs_hba *hba)
1216{
1217 if (hba->vops && hba->vops->dbg_register_dump)
1218 hba->vops->dbg_register_dump(hba);
1219}
1220
1221static inline int ufshcd_vops_device_reset(struct ufs_hba *hba)
1222{
1223 if (hba->vops && hba->vops->device_reset)
1224 return hba->vops->device_reset(hba);
1225
1226 return -EOPNOTSUPP;
1227}
1228
1229static inline void ufshcd_vops_config_scaling_param(struct ufs_hba *hba,
1230 struct devfreq_dev_profile
1231 *profile, void *data)
1232{
1233 if (hba->vops && hba->vops->config_scaling_param)
1234 hba->vops->config_scaling_param(hba, profile, data);
1235}
1236
1237extern struct ufs_pm_lvl_states ufs_pm_lvl_states[];
1238
1239
1240
1241
1242
1243
1244
1245static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
1246{
1247 if (scsi_is_wlun(scsi_lun))
1248 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
1249 | UFS_UPIU_WLUN_ID;
1250 else
1251 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
1252}
1253
1254int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
1255 const char *prefix);
1256
1257#endif
1258