1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#include <linux/async.h>
37
38#include "ufshcd.h"
39#include "unipro.h"
40
41#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
42 UTP_TASK_REQ_COMPL |\
43 UIC_POWER_MODE |\
44 UFSHCD_ERROR_MASK)
45
46#define UIC_CMD_TIMEOUT 500
47
48
49#define NOP_OUT_RETRIES 10
50
51#define NOP_OUT_TIMEOUT 30
52
53
54#define QUERY_REQ_RETRIES 10
55
56#define QUERY_REQ_TIMEOUT 30
57
58
59#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
60
61
62#define INT_AGGR_DEF_TO 0x02
63
64enum {
65 UFSHCD_MAX_CHANNEL = 0,
66 UFSHCD_MAX_ID = 1,
67 UFSHCD_MAX_LUNS = 8,
68 UFSHCD_CMD_PER_LUN = 32,
69 UFSHCD_CAN_QUEUE = 32,
70};
71
72
73enum {
74 UFSHCD_STATE_OPERATIONAL,
75 UFSHCD_STATE_RESET,
76 UFSHCD_STATE_ERROR,
77};
78
79
80enum {
81 UFSHCD_INT_DISABLE,
82 UFSHCD_INT_ENABLE,
83 UFSHCD_INT_CLEAR,
84};
85
86
87
88
89
90
91
92
93
94
95
96
97static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
98 u32 val, unsigned long interval_us, unsigned long timeout_ms)
99{
100 int err = 0;
101 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
102
103
104 val = val & mask;
105
106 while ((ufshcd_readl(hba, reg) & mask) != val) {
107
108 usleep_range(interval_us, interval_us + 50);
109
110 if (time_after(jiffies, timeout)) {
111 if ((ufshcd_readl(hba, reg) & mask) != val)
112 err = -ETIMEDOUT;
113 break;
114 }
115 }
116
117 return err;
118}
119
120
121
122
123
124
125
126static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
127{
128 if (hba->ufs_version == UFSHCI_VERSION_10)
129 return INTERRUPT_MASK_ALL_VER_10;
130 else
131 return INTERRUPT_MASK_ALL_VER_11;
132}
133
134
135
136
137
138
139
140static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
141{
142 return ufshcd_readl(hba, REG_UFS_VERSION);
143}
144
145
146
147
148
149
150
151
152static inline int ufshcd_is_device_present(u32 reg_hcs)
153{
154 return (DEVICE_PRESENT & reg_hcs) ? 1 : 0;
155}
156
157
158
159
160
161
162
163
164static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
165{
166 return lrbp->utr_descriptor_ptr->header.dword_2 & MASK_OCS;
167}
168
169
170
171
172
173
174
175
176static inline int
177ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
178{
179 return task_req_descp->header.dword_2 & MASK_OCS;
180}
181
182
183
184
185
186
187
188
189static inline int ufshcd_get_tm_free_slot(struct ufs_hba *hba)
190{
191 return find_first_zero_bit(&hba->outstanding_tasks, hba->nutmrs);
192}
193
194
195
196
197
198
199static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
200{
201 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
202}
203
204
205
206
207
208
209
210static inline int ufshcd_get_lists_status(u32 reg)
211{
212
213
214
215
216
217
218
219
220
221
222
223 return (((reg) & (0xFF)) >> 1) ^ (0x07);
224}
225
226
227
228
229
230
231
232
233static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
234{
235 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
236 MASK_UIC_COMMAND_RESULT;
237}
238
239
240
241
242
243
244
245
246static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
247{
248 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
249}
250
251
252
253
254
255static inline int
256ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
257{
258 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
259}
260
261
262
263
264
265
266
267
268static inline int
269ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
270{
271 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
272}
273
274
275
276
277
278
279
280
281static inline unsigned int
282ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
283{
284 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
285 MASK_RSP_UPIU_DATA_SEG_LEN;
286}
287
288
289
290
291
292
293
294
295
296
297static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
298{
299 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
300 MASK_RSP_EXCEPTION_EVENT ? true : false;
301}
302
303
304
305
306
307static inline void
308ufshcd_reset_intr_aggr(struct ufs_hba *hba)
309{
310 ufshcd_writel(hba, INT_AGGR_ENABLE |
311 INT_AGGR_COUNTER_AND_TIMER_RESET,
312 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
313}
314
315
316
317
318
319
320
321static inline void
322ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
323{
324 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
325 INT_AGGR_COUNTER_THLD_VAL(cnt) |
326 INT_AGGR_TIMEOUT_VAL(tmout),
327 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
328}
329
330
331
332
333
334
335
336static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
337{
338 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
339 REG_UTP_TASK_REQ_LIST_RUN_STOP);
340 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
341 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
342}
343
344
345
346
347
348static inline void ufshcd_hba_start(struct ufs_hba *hba)
349{
350 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
351}
352
353
354
355
356
357
358
359static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
360{
361 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
362}
363
364
365
366
367
368
369static inline
370void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
371{
372 __set_bit(task_tag, &hba->outstanding_reqs);
373 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
374}
375
376
377
378
379
380static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
381{
382 int len;
383 if (lrbp->sense_buffer &&
384 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
385 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
386 memcpy(lrbp->sense_buffer,
387 lrbp->ucd_rsp_ptr->sr.sense_data,
388 min_t(int, len, SCSI_SENSE_BUFFERSIZE));
389 }
390}
391
392
393
394
395
396static inline void ufshcd_query_to_cpu(struct utp_upiu_query *response)
397{
398 response->length = be16_to_cpu(response->length);
399 response->value = be32_to_cpu(response->value);
400}
401
402
403
404
405
406static inline void ufshcd_query_to_be(struct utp_upiu_query *request)
407{
408 request->length = cpu_to_be16(request->length);
409 request->value = cpu_to_be32(request->value);
410}
411
412
413
414
415
416
417
418static
419void ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
420{
421 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
422
423
424 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
425 UPIU_RSP_CODE_OFFSET;
426
427 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
428 ufshcd_query_to_cpu(&query_res->upiu_res);
429
430
431
432 if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
433 u8 *descp = (u8 *)&lrbp->ucd_rsp_ptr +
434 GENERAL_UPIU_REQUEST_SIZE;
435 u16 len;
436
437
438 len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
439 MASK_QUERY_DATA_SEG_LEN;
440
441 memcpy(hba->dev_cmd.query.descriptor, descp,
442 min_t(u16, len, QUERY_DESC_MAX_SIZE));
443 }
444}
445
446
447
448
449
450static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
451{
452 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
453
454
455 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
456 hba->nutmrs =
457 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
458}
459
460
461
462
463
464
465
466static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
467{
468 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
469 return true;
470 else
471 return false;
472}
473
474
475
476
477
478
479
480
481static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
482{
483 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
484}
485
486
487
488
489
490
491
492
493static inline void
494ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
495{
496 WARN_ON(hba->active_uic_cmd);
497
498 hba->active_uic_cmd = uic_cmd;
499
500
501 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
502 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
503 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
504
505
506 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
507 REG_UIC_COMMAND);
508}
509
510
511
512
513
514
515
516
517
518static int
519ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
520{
521 int ret;
522 unsigned long flags;
523
524 if (wait_for_completion_timeout(&uic_cmd->done,
525 msecs_to_jiffies(UIC_CMD_TIMEOUT)))
526 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
527 else
528 ret = -ETIMEDOUT;
529
530 spin_lock_irqsave(hba->host->host_lock, flags);
531 hba->active_uic_cmd = NULL;
532 spin_unlock_irqrestore(hba->host->host_lock, flags);
533
534 return ret;
535}
536
537
538
539
540
541
542
543
544
545
546static int
547__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
548{
549 int ret;
550 unsigned long flags;
551
552 if (!ufshcd_ready_for_uic_cmd(hba)) {
553 dev_err(hba->dev,
554 "Controller not ready to accept UIC commands\n");
555 return -EIO;
556 }
557
558 init_completion(&uic_cmd->done);
559
560 spin_lock_irqsave(hba->host->host_lock, flags);
561 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
562 spin_unlock_irqrestore(hba->host->host_lock, flags);
563
564 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
565
566 return ret;
567}
568
569
570
571
572
573
574
575
576static int
577ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
578{
579 int ret;
580
581 mutex_lock(&hba->uic_cmd_mutex);
582 ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
583 mutex_unlock(&hba->uic_cmd_mutex);
584
585 return ret;
586}
587
588
589
590
591
592
593
594static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
595{
596 struct ufshcd_sg_entry *prd_table;
597 struct scatterlist *sg;
598 struct scsi_cmnd *cmd;
599 int sg_segments;
600 int i;
601
602 cmd = lrbp->cmd;
603 sg_segments = scsi_dma_map(cmd);
604 if (sg_segments < 0)
605 return sg_segments;
606
607 if (sg_segments) {
608 lrbp->utr_descriptor_ptr->prd_table_length =
609 cpu_to_le16((u16) (sg_segments));
610
611 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
612
613 scsi_for_each_sg(cmd, sg, sg_segments, i) {
614 prd_table[i].size =
615 cpu_to_le32(((u32) sg_dma_len(sg))-1);
616 prd_table[i].base_addr =
617 cpu_to_le32(lower_32_bits(sg->dma_address));
618 prd_table[i].upper_addr =
619 cpu_to_le32(upper_32_bits(sg->dma_address));
620 }
621 } else {
622 lrbp->utr_descriptor_ptr->prd_table_length = 0;
623 }
624
625 return 0;
626}
627
628
629
630
631
632
633static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
634{
635 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
636
637 if (hba->ufs_version == UFSHCI_VERSION_10) {
638 u32 rw;
639 rw = set & INTERRUPT_MASK_RW_VER_10;
640 set = rw | ((set ^ intrs) & intrs);
641 } else {
642 set |= intrs;
643 }
644
645 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
646}
647
648
649
650
651
652
653static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
654{
655 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
656
657 if (hba->ufs_version == UFSHCI_VERSION_10) {
658 u32 rw;
659 rw = (set & INTERRUPT_MASK_RW_VER_10) &
660 ~(intrs & INTERRUPT_MASK_RW_VER_10);
661 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
662
663 } else {
664 set &= ~intrs;
665 }
666
667 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
668}
669
670
671
672
673
674
675
676
677static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
678 u32 *upiu_flags, enum dma_data_direction cmd_dir)
679{
680 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
681 u32 data_direction;
682 u32 dword_0;
683
684 if (cmd_dir == DMA_FROM_DEVICE) {
685 data_direction = UTP_DEVICE_TO_HOST;
686 *upiu_flags = UPIU_CMD_FLAGS_READ;
687 } else if (cmd_dir == DMA_TO_DEVICE) {
688 data_direction = UTP_HOST_TO_DEVICE;
689 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
690 } else {
691 data_direction = UTP_NO_DATA_TRANSFER;
692 *upiu_flags = UPIU_CMD_FLAGS_NONE;
693 }
694
695 dword_0 = data_direction | (lrbp->command_type
696 << UPIU_COMMAND_TYPE_OFFSET);
697 if (lrbp->intr_cmd)
698 dword_0 |= UTP_REQ_DESC_INT_CMD;
699
700
701 req_desc->header.dword_0 = cpu_to_le32(dword_0);
702
703
704
705
706
707
708 req_desc->header.dword_2 =
709 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
710}
711
712
713
714
715
716
717
718static
719void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
720{
721 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
722
723
724 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
725 UPIU_TRANSACTION_COMMAND, upiu_flags,
726 lrbp->lun, lrbp->task_tag);
727 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
728 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
729
730
731 ucd_req_ptr->header.dword_2 = 0;
732
733 ucd_req_ptr->sc.exp_data_transfer_len =
734 cpu_to_be32(lrbp->cmd->sdb.length);
735
736 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd,
737 (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE)));
738}
739
740
741
742
743
744
745
746
747static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
748 struct ufshcd_lrb *lrbp, u32 upiu_flags)
749{
750 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
751 struct ufs_query *query = &hba->dev_cmd.query;
752 u16 len = query->request.upiu_req.length;
753 u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
754
755
756 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
757 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
758 lrbp->lun, lrbp->task_tag);
759 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
760 0, query->request.query_func, 0, 0);
761
762
763 ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD(
764 0, 0, len >> 8, (u8)len);
765
766
767 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
768 QUERY_OSF_SIZE);
769 ufshcd_query_to_be(&ucd_req_ptr->qr);
770
771
772 if ((len > 0) && (query->request.upiu_req.opcode ==
773 UPIU_QUERY_OPCODE_WRITE_DESC)) {
774 memcpy(descp, query->descriptor,
775 min_t(u16, len, QUERY_DESC_MAX_SIZE));
776 }
777}
778
779static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
780{
781 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
782
783 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
784
785
786 ucd_req_ptr->header.dword_0 =
787 UPIU_HEADER_DWORD(
788 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
789}
790
791
792
793
794
795
796static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
797{
798 u32 upiu_flags;
799 int ret = 0;
800
801 switch (lrbp->command_type) {
802 case UTP_CMD_TYPE_SCSI:
803 if (likely(lrbp->cmd)) {
804 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
805 lrbp->cmd->sc_data_direction);
806 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
807 } else {
808 ret = -EINVAL;
809 }
810 break;
811 case UTP_CMD_TYPE_DEV_MANAGE:
812 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
813 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
814 ufshcd_prepare_utp_query_req_upiu(
815 hba, lrbp, upiu_flags);
816 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
817 ufshcd_prepare_utp_nop_upiu(lrbp);
818 else
819 ret = -EINVAL;
820 break;
821 case UTP_CMD_TYPE_UFS:
822
823 ret = -ENOTSUPP;
824 dev_err(hba->dev, "%s: UFS native command are not supported\n",
825 __func__);
826 break;
827 default:
828 ret = -ENOTSUPP;
829 dev_err(hba->dev, "%s: unknown command type: 0x%x\n",
830 __func__, lrbp->command_type);
831 break;
832 }
833
834 return ret;
835}
836
837
838
839
840
841
842
843
844static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
845{
846 struct ufshcd_lrb *lrbp;
847 struct ufs_hba *hba;
848 unsigned long flags;
849 int tag;
850 int err = 0;
851
852 hba = shost_priv(host);
853
854 tag = cmd->request->tag;
855
856 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
857 err = SCSI_MLQUEUE_HOST_BUSY;
858 goto out;
859 }
860
861
862 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
863
864
865
866
867
868
869 err = SCSI_MLQUEUE_HOST_BUSY;
870 goto out;
871 }
872
873 lrbp = &hba->lrb[tag];
874
875 WARN_ON(lrbp->cmd);
876 lrbp->cmd = cmd;
877 lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
878 lrbp->sense_buffer = cmd->sense_buffer;
879 lrbp->task_tag = tag;
880 lrbp->lun = cmd->device->lun;
881 lrbp->intr_cmd = false;
882 lrbp->command_type = UTP_CMD_TYPE_SCSI;
883
884
885 ufshcd_compose_upiu(hba, lrbp);
886 err = ufshcd_map_sg(lrbp);
887 if (err) {
888 lrbp->cmd = NULL;
889 clear_bit_unlock(tag, &hba->lrb_in_use);
890 goto out;
891 }
892
893
894 spin_lock_irqsave(hba->host->host_lock, flags);
895 ufshcd_send_command(hba, tag);
896 spin_unlock_irqrestore(hba->host->host_lock, flags);
897out:
898 return err;
899}
900
901static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
902 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
903{
904 lrbp->cmd = NULL;
905 lrbp->sense_bufflen = 0;
906 lrbp->sense_buffer = NULL;
907 lrbp->task_tag = tag;
908 lrbp->lun = 0;
909 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
910 lrbp->intr_cmd = true;
911 hba->dev_cmd.type = cmd_type;
912
913 return ufshcd_compose_upiu(hba, lrbp);
914}
915
916static int
917ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
918{
919 int err = 0;
920 unsigned long flags;
921 u32 mask = 1 << tag;
922
923
924 spin_lock_irqsave(hba->host->host_lock, flags);
925 ufshcd_utrl_clear(hba, tag);
926 spin_unlock_irqrestore(hba->host->host_lock, flags);
927
928
929
930
931
932 err = ufshcd_wait_for_register(hba,
933 REG_UTP_TRANSFER_REQ_DOOR_BELL,
934 mask, ~mask, 1000, 1000);
935
936 return err;
937}
938
939
940
941
942
943
944static int
945ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
946{
947 int resp;
948 int err = 0;
949
950 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
951
952 switch (resp) {
953 case UPIU_TRANSACTION_NOP_IN:
954 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
955 err = -EINVAL;
956 dev_err(hba->dev, "%s: unexpected response %x\n",
957 __func__, resp);
958 }
959 break;
960 case UPIU_TRANSACTION_QUERY_RSP:
961 ufshcd_copy_query_response(hba, lrbp);
962 break;
963 case UPIU_TRANSACTION_REJECT_UPIU:
964
965 err = -EPERM;
966 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
967 __func__);
968 break;
969 default:
970 err = -EINVAL;
971 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
972 __func__, resp);
973 break;
974 }
975
976 return err;
977}
978
979static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
980 struct ufshcd_lrb *lrbp, int max_timeout)
981{
982 int err = 0;
983 unsigned long time_left;
984 unsigned long flags;
985
986 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
987 msecs_to_jiffies(max_timeout));
988
989 spin_lock_irqsave(hba->host->host_lock, flags);
990 hba->dev_cmd.complete = NULL;
991 if (likely(time_left)) {
992 err = ufshcd_get_tr_ocs(lrbp);
993 if (!err)
994 err = ufshcd_dev_cmd_completion(hba, lrbp);
995 }
996 spin_unlock_irqrestore(hba->host->host_lock, flags);
997
998 if (!time_left) {
999 err = -ETIMEDOUT;
1000 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
1001
1002 err = -EAGAIN;
1003 }
1004
1005 return err;
1006}
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
1020{
1021 int tag;
1022 bool ret = false;
1023 unsigned long tmp;
1024
1025 if (!tag_out)
1026 goto out;
1027
1028 do {
1029 tmp = ~hba->lrb_in_use;
1030 tag = find_last_bit(&tmp, hba->nutrs);
1031 if (tag >= hba->nutrs)
1032 goto out;
1033 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
1034
1035 *tag_out = tag;
1036 ret = true;
1037out:
1038 return ret;
1039}
1040
1041static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
1042{
1043 clear_bit_unlock(tag, &hba->lrb_in_use);
1044}
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
1056 enum dev_cmd_type cmd_type, int timeout)
1057{
1058 struct ufshcd_lrb *lrbp;
1059 int err;
1060 int tag;
1061 struct completion wait;
1062 unsigned long flags;
1063
1064
1065
1066
1067
1068
1069 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
1070
1071 init_completion(&wait);
1072 lrbp = &hba->lrb[tag];
1073 WARN_ON(lrbp->cmd);
1074 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
1075 if (unlikely(err))
1076 goto out_put_tag;
1077
1078 hba->dev_cmd.complete = &wait;
1079
1080 spin_lock_irqsave(hba->host->host_lock, flags);
1081 ufshcd_send_command(hba, tag);
1082 spin_unlock_irqrestore(hba->host->host_lock, flags);
1083
1084 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
1085
1086out_put_tag:
1087 ufshcd_put_dev_cmd_tag(hba, tag);
1088 wake_up(&hba->dev_cmd.tag_wq);
1089 return err;
1090}
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1102 enum flag_idn idn, bool *flag_res)
1103{
1104 struct ufs_query_req *request;
1105 struct ufs_query_res *response;
1106 int err;
1107
1108 BUG_ON(!hba);
1109
1110 mutex_lock(&hba->dev_cmd.lock);
1111 request = &hba->dev_cmd.query.request;
1112 response = &hba->dev_cmd.query.response;
1113 memset(request, 0, sizeof(struct ufs_query_req));
1114 memset(response, 0, sizeof(struct ufs_query_res));
1115
1116 switch (opcode) {
1117 case UPIU_QUERY_OPCODE_SET_FLAG:
1118 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1119 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1120 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1121 break;
1122 case UPIU_QUERY_OPCODE_READ_FLAG:
1123 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1124 if (!flag_res) {
1125
1126 dev_err(hba->dev, "%s: Invalid argument for read request\n",
1127 __func__);
1128 err = -EINVAL;
1129 goto out_unlock;
1130 }
1131 break;
1132 default:
1133 dev_err(hba->dev,
1134 "%s: Expected query flag opcode but got = %d\n",
1135 __func__, opcode);
1136 err = -EINVAL;
1137 goto out_unlock;
1138 }
1139 request->upiu_req.opcode = opcode;
1140 request->upiu_req.idn = idn;
1141
1142
1143 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY,
1144 QUERY_REQ_TIMEOUT);
1145
1146 if (err) {
1147 dev_err(hba->dev,
1148 "%s: Sending flag query for idn %d failed, err = %d\n",
1149 __func__, idn, err);
1150 goto out_unlock;
1151 }
1152
1153 if (flag_res)
1154 *flag_res = (response->upiu_res.value &
1155 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1156
1157out_unlock:
1158 mutex_unlock(&hba->dev_cmd.lock);
1159 return err;
1160}
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1174 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
1175{
1176 struct ufs_query_req *request;
1177 struct ufs_query_res *response;
1178 int err;
1179
1180 BUG_ON(!hba);
1181
1182 if (!attr_val) {
1183 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
1184 __func__, opcode);
1185 err = -EINVAL;
1186 goto out;
1187 }
1188
1189 mutex_lock(&hba->dev_cmd.lock);
1190 request = &hba->dev_cmd.query.request;
1191 response = &hba->dev_cmd.query.response;
1192 memset(request, 0, sizeof(struct ufs_query_req));
1193 memset(response, 0, sizeof(struct ufs_query_res));
1194
1195 switch (opcode) {
1196 case UPIU_QUERY_OPCODE_WRITE_ATTR:
1197 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1198 request->upiu_req.value = *attr_val;
1199 break;
1200 case UPIU_QUERY_OPCODE_READ_ATTR:
1201 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1202 break;
1203 default:
1204 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
1205 __func__, opcode);
1206 err = -EINVAL;
1207 goto out_unlock;
1208 }
1209
1210 request->upiu_req.opcode = opcode;
1211 request->upiu_req.idn = idn;
1212 request->upiu_req.index = index;
1213 request->upiu_req.selector = selector;
1214
1215
1216 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY,
1217 QUERY_REQ_TIMEOUT);
1218
1219 if (err) {
1220 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1221 __func__, opcode, idn, err);
1222 goto out_unlock;
1223 }
1224
1225 *attr_val = response->upiu_res.value;
1226
1227out_unlock:
1228 mutex_unlock(&hba->dev_cmd.lock);
1229out:
1230 return err;
1231}
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246static int ufshcd_memory_alloc(struct ufs_hba *hba)
1247{
1248 size_t utmrdl_size, utrdl_size, ucdl_size;
1249
1250
1251 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
1252 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
1253 ucdl_size,
1254 &hba->ucdl_dma_addr,
1255 GFP_KERNEL);
1256
1257
1258
1259
1260
1261
1262
1263 if (!hba->ucdl_base_addr ||
1264 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
1265 dev_err(hba->dev,
1266 "Command Descriptor Memory allocation failed\n");
1267 goto out;
1268 }
1269
1270
1271
1272
1273
1274 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
1275 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
1276 utrdl_size,
1277 &hba->utrdl_dma_addr,
1278 GFP_KERNEL);
1279 if (!hba->utrdl_base_addr ||
1280 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
1281 dev_err(hba->dev,
1282 "Transfer Descriptor Memory allocation failed\n");
1283 goto out;
1284 }
1285
1286
1287
1288
1289
1290 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
1291 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
1292 utmrdl_size,
1293 &hba->utmrdl_dma_addr,
1294 GFP_KERNEL);
1295 if (!hba->utmrdl_base_addr ||
1296 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
1297 dev_err(hba->dev,
1298 "Task Management Descriptor Memory allocation failed\n");
1299 goto out;
1300 }
1301
1302
1303 hba->lrb = devm_kzalloc(hba->dev,
1304 hba->nutrs * sizeof(struct ufshcd_lrb),
1305 GFP_KERNEL);
1306 if (!hba->lrb) {
1307 dev_err(hba->dev, "LRB Memory allocation failed\n");
1308 goto out;
1309 }
1310 return 0;
1311out:
1312 return -ENOMEM;
1313}
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328static void ufshcd_host_memory_configure(struct ufs_hba *hba)
1329{
1330 struct utp_transfer_cmd_desc *cmd_descp;
1331 struct utp_transfer_req_desc *utrdlp;
1332 dma_addr_t cmd_desc_dma_addr;
1333 dma_addr_t cmd_desc_element_addr;
1334 u16 response_offset;
1335 u16 prdt_offset;
1336 int cmd_desc_size;
1337 int i;
1338
1339 utrdlp = hba->utrdl_base_addr;
1340 cmd_descp = hba->ucdl_base_addr;
1341
1342 response_offset =
1343 offsetof(struct utp_transfer_cmd_desc, response_upiu);
1344 prdt_offset =
1345 offsetof(struct utp_transfer_cmd_desc, prd_table);
1346
1347 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
1348 cmd_desc_dma_addr = hba->ucdl_dma_addr;
1349
1350 for (i = 0; i < hba->nutrs; i++) {
1351
1352 cmd_desc_element_addr =
1353 (cmd_desc_dma_addr + (cmd_desc_size * i));
1354 utrdlp[i].command_desc_base_addr_lo =
1355 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
1356 utrdlp[i].command_desc_base_addr_hi =
1357 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
1358
1359
1360 utrdlp[i].response_upiu_offset =
1361 cpu_to_le16((response_offset >> 2));
1362 utrdlp[i].prd_table_offset =
1363 cpu_to_le16((prdt_offset >> 2));
1364 utrdlp[i].response_upiu_length =
1365 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
1366
1367 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
1368 hba->lrb[i].ucd_req_ptr =
1369 (struct utp_upiu_req *)(cmd_descp + i);
1370 hba->lrb[i].ucd_rsp_ptr =
1371 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
1372 hba->lrb[i].ucd_prdt_ptr =
1373 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
1374 }
1375}
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388static int ufshcd_dme_link_startup(struct ufs_hba *hba)
1389{
1390 struct uic_command uic_cmd = {0};
1391 int ret;
1392
1393 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
1394
1395 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
1396 if (ret)
1397 dev_err(hba->dev,
1398 "dme-link-startup: error code %d\n", ret);
1399 return ret;
1400}
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
1413 u8 attr_set, u32 mib_val, u8 peer)
1414{
1415 struct uic_command uic_cmd = {0};
1416 static const char *const action[] = {
1417 "dme-set",
1418 "dme-peer-set"
1419 };
1420 const char *set = action[!!peer];
1421 int ret;
1422
1423 uic_cmd.command = peer ?
1424 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
1425 uic_cmd.argument1 = attr_sel;
1426 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
1427 uic_cmd.argument3 = mib_val;
1428
1429 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
1430 if (ret)
1431 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
1432 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
1433
1434 return ret;
1435}
1436EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
1448 u32 *mib_val, u8 peer)
1449{
1450 struct uic_command uic_cmd = {0};
1451 static const char *const action[] = {
1452 "dme-get",
1453 "dme-peer-get"
1454 };
1455 const char *get = action[!!peer];
1456 int ret;
1457
1458 uic_cmd.command = peer ?
1459 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
1460 uic_cmd.argument1 = attr_sel;
1461
1462 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
1463 if (ret) {
1464 dev_err(hba->dev, "%s: attr-id 0x%x error code %d\n",
1465 get, UIC_GET_ATTR_ID(attr_sel), ret);
1466 goto out;
1467 }
1468
1469 if (mib_val)
1470 *mib_val = uic_cmd.argument3;
1471out:
1472 return ret;
1473}
1474EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
1485{
1486 struct uic_command uic_cmd = {0};
1487 struct completion pwr_done;
1488 unsigned long flags;
1489 u8 status;
1490 int ret;
1491
1492 uic_cmd.command = UIC_CMD_DME_SET;
1493 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
1494 uic_cmd.argument3 = mode;
1495 init_completion(&pwr_done);
1496
1497 mutex_lock(&hba->uic_cmd_mutex);
1498
1499 spin_lock_irqsave(hba->host->host_lock, flags);
1500 hba->pwr_done = &pwr_done;
1501 spin_unlock_irqrestore(hba->host->host_lock, flags);
1502 ret = __ufshcd_send_uic_cmd(hba, &uic_cmd);
1503 if (ret) {
1504 dev_err(hba->dev,
1505 "pwr mode change with mode 0x%x uic error %d\n",
1506 mode, ret);
1507 goto out;
1508 }
1509
1510 if (!wait_for_completion_timeout(hba->pwr_done,
1511 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
1512 dev_err(hba->dev,
1513 "pwr mode change with mode 0x%x completion timeout\n",
1514 mode);
1515 ret = -ETIMEDOUT;
1516 goto out;
1517 }
1518
1519 status = ufshcd_get_upmcrs(hba);
1520 if (status != PWR_LOCAL) {
1521 dev_err(hba->dev,
1522 "pwr mode change failed, host umpcrs:0x%x\n",
1523 status);
1524 ret = (status != PWR_OK) ? status : -1;
1525 }
1526out:
1527 spin_lock_irqsave(hba->host->host_lock, flags);
1528 hba->pwr_done = NULL;
1529 spin_unlock_irqrestore(hba->host->host_lock, flags);
1530 mutex_unlock(&hba->uic_cmd_mutex);
1531 return ret;
1532}
1533
1534
1535
1536
1537
1538
1539
1540
1541static int ufshcd_config_max_pwr_mode(struct ufs_hba *hba)
1542{
1543 enum {RX = 0, TX = 1};
1544 u32 lanes[] = {1, 1};
1545 u32 gear[] = {1, 1};
1546 u8 pwr[] = {FASTAUTO_MODE, FASTAUTO_MODE};
1547 int ret;
1548
1549
1550 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), &lanes[RX]);
1551 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), &lanes[TX]);
1552
1553
1554
1555
1556
1557
1558 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &gear[RX]);
1559 if (!gear[RX]) {
1560 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), &gear[RX]);
1561 pwr[RX] = SLOWAUTO_MODE;
1562 }
1563
1564 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &gear[TX]);
1565 if (!gear[TX]) {
1566 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
1567 &gear[TX]);
1568 pwr[TX] = SLOWAUTO_MODE;
1569 }
1570
1571
1572
1573
1574
1575
1576
1577 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), gear[RX]);
1578 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), lanes[RX]);
1579 if (pwr[RX] == FASTAUTO_MODE)
1580 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
1581
1582 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), gear[TX]);
1583 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), lanes[TX]);
1584 if (pwr[TX] == FASTAUTO_MODE)
1585 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
1586
1587 if (pwr[RX] == FASTAUTO_MODE || pwr[TX] == FASTAUTO_MODE)
1588 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), PA_HS_MODE_B);
1589
1590 ret = ufshcd_uic_change_pwr_mode(hba, pwr[RX] << 4 | pwr[TX]);
1591 if (ret)
1592 dev_err(hba->dev,
1593 "pwr_mode: power mode change failed %d\n", ret);
1594
1595 return ret;
1596}
1597
1598
1599
1600
1601
1602
1603
1604static int ufshcd_complete_dev_init(struct ufs_hba *hba)
1605{
1606 int i, retries, err = 0;
1607 bool flag_res = 1;
1608
1609 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1610
1611 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1612 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
1613 if (!err || err == -ETIMEDOUT)
1614 break;
1615 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1616 }
1617 if (err) {
1618 dev_err(hba->dev,
1619 "%s setting fDeviceInit flag failed with error %d\n",
1620 __func__, err);
1621 goto out;
1622 }
1623
1624
1625 for (i = 0; i < 100 && !err && flag_res; i++) {
1626 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1627 err = ufshcd_query_flag(hba,
1628 UPIU_QUERY_OPCODE_READ_FLAG,
1629 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
1630 if (!err || err == -ETIMEDOUT)
1631 break;
1632 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__,
1633 err);
1634 }
1635 }
1636 if (err)
1637 dev_err(hba->dev,
1638 "%s reading fDeviceInit flag failed with error %d\n",
1639 __func__, err);
1640 else if (flag_res)
1641 dev_err(hba->dev,
1642 "%s fDeviceInit was not cleared by the device\n",
1643 __func__);
1644
1645out:
1646 return err;
1647}
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662static int ufshcd_make_hba_operational(struct ufs_hba *hba)
1663{
1664 int err = 0;
1665 u32 reg;
1666
1667
1668 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
1669 if (!ufshcd_is_device_present(reg)) {
1670 dev_err(hba->dev, "cc: Device not present\n");
1671 err = -ENXIO;
1672 goto out;
1673 }
1674
1675
1676 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
1677
1678
1679 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
1680
1681
1682 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
1683 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
1684 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
1685 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
1686 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
1687 REG_UTP_TASK_REQ_LIST_BASE_L);
1688 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
1689 REG_UTP_TASK_REQ_LIST_BASE_H);
1690
1691
1692
1693
1694
1695 if (!(ufshcd_get_lists_status(reg))) {
1696 ufshcd_enable_run_stop_reg(hba);
1697 } else {
1698 dev_err(hba->dev,
1699 "Host controller not ready to process requests");
1700 err = -EIO;
1701 goto out;
1702 }
1703
1704 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
1705 scsi_unblock_requests(hba->host);
1706
1707 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
1708
1709out:
1710 return err;
1711}
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723static int ufshcd_hba_enable(struct ufs_hba *hba)
1724{
1725 int retry;
1726
1727
1728
1729
1730
1731
1732
1733 if (!ufshcd_is_hba_active(hba)) {
1734
1735
1736 ufshcd_hba_stop(hba);
1737
1738
1739
1740
1741
1742
1743 msleep(5);
1744 }
1745
1746
1747 ufshcd_hba_start(hba);
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759 msleep(1);
1760
1761
1762 retry = 10;
1763 while (ufshcd_is_hba_active(hba)) {
1764 if (retry) {
1765 retry--;
1766 } else {
1767 dev_err(hba->dev,
1768 "Controller enable failed\n");
1769 return -EIO;
1770 }
1771 msleep(5);
1772 }
1773 return 0;
1774}
1775
1776
1777
1778
1779
1780
1781
1782static int ufshcd_link_startup(struct ufs_hba *hba)
1783{
1784 int ret;
1785
1786
1787 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
1788
1789 ret = ufshcd_dme_link_startup(hba);
1790 if (ret)
1791 goto out;
1792
1793 ret = ufshcd_make_hba_operational(hba);
1794
1795out:
1796 if (ret)
1797 dev_err(hba->dev, "link startup failed %d\n", ret);
1798 return ret;
1799}
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811static int ufshcd_verify_dev_init(struct ufs_hba *hba)
1812{
1813 int err = 0;
1814 int retries;
1815
1816 mutex_lock(&hba->dev_cmd.lock);
1817 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
1818 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1819 NOP_OUT_TIMEOUT);
1820
1821 if (!err || err == -ETIMEDOUT)
1822 break;
1823
1824 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1825 }
1826 mutex_unlock(&hba->dev_cmd.lock);
1827
1828 if (err)
1829 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
1830 return err;
1831}
1832
1833
1834
1835
1836
1837
1838
1839static int ufshcd_do_reset(struct ufs_hba *hba)
1840{
1841 struct ufshcd_lrb *lrbp;
1842 unsigned long flags;
1843 int tag;
1844
1845
1846 scsi_block_requests(hba->host);
1847
1848 spin_lock_irqsave(hba->host->host_lock, flags);
1849 hba->ufshcd_state = UFSHCD_STATE_RESET;
1850
1851
1852 ufshcd_hba_stop(hba);
1853 spin_unlock_irqrestore(hba->host->host_lock, flags);
1854
1855
1856 for (tag = 0; tag < hba->nutrs; tag++) {
1857 if (test_bit(tag, &hba->outstanding_reqs)) {
1858 lrbp = &hba->lrb[tag];
1859 if (lrbp->cmd) {
1860 scsi_dma_unmap(lrbp->cmd);
1861 lrbp->cmd->result = DID_RESET << 16;
1862 lrbp->cmd->scsi_done(lrbp->cmd);
1863 lrbp->cmd = NULL;
1864 clear_bit_unlock(tag, &hba->lrb_in_use);
1865 }
1866 }
1867 }
1868
1869
1870 if (hba->dev_cmd.complete)
1871 complete(hba->dev_cmd.complete);
1872
1873
1874 hba->outstanding_reqs = 0;
1875 hba->outstanding_tasks = 0;
1876
1877
1878 if (ufshcd_hba_enable(hba)) {
1879 dev_err(hba->dev,
1880 "Reset: Controller initialization failed\n");
1881 return FAILED;
1882 }
1883
1884 if (ufshcd_link_startup(hba)) {
1885 dev_err(hba->dev,
1886 "Reset: Link start-up failed\n");
1887 return FAILED;
1888 }
1889
1890 return SUCCESS;
1891}
1892
1893
1894
1895
1896
1897
1898
1899static int ufshcd_slave_alloc(struct scsi_device *sdev)
1900{
1901 struct ufs_hba *hba;
1902
1903 hba = shost_priv(sdev->host);
1904 sdev->tagged_supported = 1;
1905
1906
1907 sdev->use_10_for_ms = 1;
1908 scsi_set_tag_type(sdev, MSG_SIMPLE_TAG);
1909
1910
1911
1912
1913
1914
1915
1916
1917 scsi_activate_tcq(sdev, hba->nutrs);
1918 return 0;
1919}
1920
1921
1922
1923
1924
1925static void ufshcd_slave_destroy(struct scsi_device *sdev)
1926{
1927 struct ufs_hba *hba;
1928
1929 hba = shost_priv(sdev->host);
1930 scsi_deactivate_tcq(sdev, hba->nutrs);
1931}
1932
1933
1934
1935
1936
1937
1938
1939
1940static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index)
1941{
1942 struct utp_task_req_desc *task_req_descp;
1943 struct utp_upiu_task_rsp *task_rsp_upiup;
1944 unsigned long flags;
1945 int ocs_value;
1946 int task_result;
1947
1948 spin_lock_irqsave(hba->host->host_lock, flags);
1949
1950
1951 __clear_bit(index, &hba->outstanding_tasks);
1952
1953 task_req_descp = hba->utmrdl_base_addr;
1954 ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
1955
1956 if (ocs_value == OCS_SUCCESS) {
1957 task_rsp_upiup = (struct utp_upiu_task_rsp *)
1958 task_req_descp[index].task_rsp_upiu;
1959 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
1960 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
1961
1962 if (task_result != UPIU_TASK_MANAGEMENT_FUNC_COMPL &&
1963 task_result != UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED)
1964 task_result = FAILED;
1965 else
1966 task_result = SUCCESS;
1967 } else {
1968 task_result = FAILED;
1969 dev_err(hba->dev,
1970 "trc: Invalid ocs = %x\n", ocs_value);
1971 }
1972 spin_unlock_irqrestore(hba->host->host_lock, flags);
1973 return task_result;
1974}
1975
1976
1977
1978
1979
1980
1981static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd *cmd)
1982{
1983 struct ufs_hba *hba;
1984 int i;
1985 int lun_qdepth = 0;
1986
1987 hba = shost_priv(cmd->device->host);
1988
1989
1990
1991
1992
1993 for (i = 0; i < hba->nutrs; i++) {
1994 if (test_bit(i, &hba->outstanding_reqs)) {
1995
1996
1997
1998
1999
2000 if (cmd->device->lun == hba->lrb[i].lun)
2001 lun_qdepth++;
2002 }
2003 }
2004
2005
2006
2007
2008
2009 scsi_adjust_queue_depth(cmd->device, MSG_SIMPLE_TAG, lun_qdepth - 1);
2010}
2011
2012
2013
2014
2015
2016
2017
2018
2019static inline int
2020ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
2021{
2022 int result = 0;
2023
2024 switch (scsi_status) {
2025 case SAM_STAT_CHECK_CONDITION:
2026 ufshcd_copy_sense_data(lrbp);
2027 case SAM_STAT_GOOD:
2028 result |= DID_OK << 16 |
2029 COMMAND_COMPLETE << 8 |
2030 scsi_status;
2031 break;
2032 case SAM_STAT_TASK_SET_FULL:
2033
2034
2035
2036
2037
2038 ufshcd_adjust_lun_qdepth(lrbp->cmd);
2039 case SAM_STAT_BUSY:
2040 case SAM_STAT_TASK_ABORTED:
2041 ufshcd_copy_sense_data(lrbp);
2042 result |= scsi_status;
2043 break;
2044 default:
2045 result |= DID_ERROR << 16;
2046 break;
2047 }
2048
2049 return result;
2050}
2051
2052
2053
2054
2055
2056
2057
2058
2059static inline int
2060ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2061{
2062 int result = 0;
2063 int scsi_status;
2064 int ocs;
2065
2066
2067 ocs = ufshcd_get_tr_ocs(lrbp);
2068
2069 switch (ocs) {
2070 case OCS_SUCCESS:
2071 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2072
2073 switch (result) {
2074 case UPIU_TRANSACTION_RESPONSE:
2075
2076
2077
2078
2079 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
2080
2081
2082
2083
2084
2085 scsi_status = result & MASK_SCSI_STATUS;
2086 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
2087
2088 if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
2089 schedule_work(&hba->eeh_work);
2090 break;
2091 case UPIU_TRANSACTION_REJECT_UPIU:
2092
2093 result = DID_ERROR << 16;
2094 dev_err(hba->dev,
2095 "Reject UPIU not fully implemented\n");
2096 break;
2097 default:
2098 result = DID_ERROR << 16;
2099 dev_err(hba->dev,
2100 "Unexpected request response code = %x\n",
2101 result);
2102 break;
2103 }
2104 break;
2105 case OCS_ABORTED:
2106 result |= DID_ABORT << 16;
2107 break;
2108 case OCS_INVALID_CMD_TABLE_ATTR:
2109 case OCS_INVALID_PRDT_ATTR:
2110 case OCS_MISMATCH_DATA_BUF_SIZE:
2111 case OCS_MISMATCH_RESP_UPIU_SIZE:
2112 case OCS_PEER_COMM_FAILURE:
2113 case OCS_FATAL_ERROR:
2114 default:
2115 result |= DID_ERROR << 16;
2116 dev_err(hba->dev,
2117 "OCS error from controller = %x\n", ocs);
2118 break;
2119 }
2120
2121 return result;
2122}
2123
2124
2125
2126
2127
2128
2129static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
2130{
2131 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
2132 hba->active_uic_cmd->argument2 |=
2133 ufshcd_get_uic_cmd_result(hba);
2134 hba->active_uic_cmd->argument3 =
2135 ufshcd_get_dme_attr_val(hba);
2136 complete(&hba->active_uic_cmd->done);
2137 }
2138
2139 if ((intr_status & UIC_POWER_MODE) && hba->pwr_done)
2140 complete(hba->pwr_done);
2141}
2142
2143
2144
2145
2146
2147static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
2148{
2149 struct ufshcd_lrb *lrbp;
2150 struct scsi_cmnd *cmd;
2151 unsigned long completed_reqs;
2152 u32 tr_doorbell;
2153 int result;
2154 int index;
2155 bool int_aggr_reset = false;
2156
2157 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
2158 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
2159
2160 for (index = 0; index < hba->nutrs; index++) {
2161 if (test_bit(index, &completed_reqs)) {
2162 lrbp = &hba->lrb[index];
2163 cmd = lrbp->cmd;
2164
2165
2166
2167
2168 int_aggr_reset |= !lrbp->intr_cmd;
2169
2170 if (cmd) {
2171 result = ufshcd_transfer_rsp_status(hba, lrbp);
2172 scsi_dma_unmap(cmd);
2173 cmd->result = result;
2174
2175 lrbp->cmd = NULL;
2176 clear_bit_unlock(index, &hba->lrb_in_use);
2177
2178 cmd->scsi_done(cmd);
2179 } else if (lrbp->command_type ==
2180 UTP_CMD_TYPE_DEV_MANAGE) {
2181 if (hba->dev_cmd.complete)
2182 complete(hba->dev_cmd.complete);
2183 }
2184 }
2185 }
2186
2187
2188 hba->outstanding_reqs ^= completed_reqs;
2189
2190
2191 wake_up(&hba->dev_cmd.tag_wq);
2192
2193
2194 if (int_aggr_reset)
2195 ufshcd_reset_intr_aggr(hba);
2196}
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
2209{
2210 int err = 0;
2211 u32 val;
2212
2213 if (!(hba->ee_ctrl_mask & mask))
2214 goto out;
2215
2216 val = hba->ee_ctrl_mask & ~mask;
2217 val &= 0xFFFF;
2218 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
2219 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
2220 if (!err)
2221 hba->ee_ctrl_mask &= ~mask;
2222out:
2223 return err;
2224}
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
2237{
2238 int err = 0;
2239 u32 val;
2240
2241 if (hba->ee_ctrl_mask & mask)
2242 goto out;
2243
2244 val = hba->ee_ctrl_mask | mask;
2245 val &= 0xFFFF;
2246 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
2247 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
2248 if (!err)
2249 hba->ee_ctrl_mask |= mask;
2250out:
2251 return err;
2252}
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
2266{
2267 int err = 0;
2268
2269 if (hba->auto_bkops_enabled)
2270 goto out;
2271
2272 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG,
2273 QUERY_FLAG_IDN_BKOPS_EN, NULL);
2274 if (err) {
2275 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
2276 __func__, err);
2277 goto out;
2278 }
2279
2280 hba->auto_bkops_enabled = true;
2281
2282
2283 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
2284 if (err)
2285 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
2286 __func__, err);
2287out:
2288 return err;
2289}
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
2304{
2305 int err = 0;
2306
2307 if (!hba->auto_bkops_enabled)
2308 goto out;
2309
2310
2311
2312
2313
2314 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
2315 if (err) {
2316 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
2317 __func__, err);
2318 goto out;
2319 }
2320
2321 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
2322 QUERY_FLAG_IDN_BKOPS_EN, NULL);
2323 if (err) {
2324 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
2325 __func__, err);
2326 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
2327 goto out;
2328 }
2329
2330 hba->auto_bkops_enabled = false;
2331out:
2332 return err;
2333}
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
2344{
2345 hba->auto_bkops_enabled = false;
2346 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
2347 ufshcd_enable_auto_bkops(hba);
2348}
2349
2350static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
2351{
2352 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
2353 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
2354}
2355
2356
2357
2358
2359
2360
2361
2362
2363static int ufshcd_urgent_bkops(struct ufs_hba *hba)
2364{
2365 int err;
2366 u32 status = 0;
2367
2368 err = ufshcd_get_bkops_status(hba, &status);
2369 if (err) {
2370 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
2371 __func__, err);
2372 goto out;
2373 }
2374
2375 status = status & 0xF;
2376
2377
2378 if (status >= BKOPS_STATUS_PERF_IMPACT)
2379 err = ufshcd_enable_auto_bkops(hba);
2380out:
2381 return err;
2382}
2383
2384static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
2385{
2386 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
2387 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
2388}
2389
2390
2391
2392
2393
2394
2395
2396
2397static void ufshcd_exception_event_handler(struct work_struct *work)
2398{
2399 struct ufs_hba *hba;
2400 int err;
2401 u32 status = 0;
2402 hba = container_of(work, struct ufs_hba, eeh_work);
2403
2404 pm_runtime_get_sync(hba->dev);
2405 err = ufshcd_get_ee_status(hba, &status);
2406 if (err) {
2407 dev_err(hba->dev, "%s: failed to get exception status %d\n",
2408 __func__, err);
2409 goto out;
2410 }
2411
2412 status &= hba->ee_ctrl_mask;
2413 if (status & MASK_EE_URGENT_BKOPS) {
2414 err = ufshcd_urgent_bkops(hba);
2415 if (err)
2416 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
2417 __func__, err);
2418 }
2419out:
2420 pm_runtime_put_sync(hba->dev);
2421 return;
2422}
2423
2424
2425
2426
2427
2428static void ufshcd_fatal_err_handler(struct work_struct *work)
2429{
2430 struct ufs_hba *hba;
2431 hba = container_of(work, struct ufs_hba, feh_workq);
2432
2433 pm_runtime_get_sync(hba->dev);
2434
2435 if (hba->ufshcd_state != UFSHCD_STATE_RESET)
2436 ufshcd_do_reset(hba);
2437 pm_runtime_put_sync(hba->dev);
2438}
2439
2440
2441
2442
2443
2444static void ufshcd_err_handler(struct ufs_hba *hba)
2445{
2446 u32 reg;
2447
2448 if (hba->errors & INT_FATAL_ERRORS)
2449 goto fatal_eh;
2450
2451 if (hba->errors & UIC_ERROR) {
2452 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
2453 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
2454 goto fatal_eh;
2455 }
2456 return;
2457fatal_eh:
2458 hba->ufshcd_state = UFSHCD_STATE_ERROR;
2459 schedule_work(&hba->feh_workq);
2460}
2461
2462
2463
2464
2465
2466static void ufshcd_tmc_handler(struct ufs_hba *hba)
2467{
2468 u32 tm_doorbell;
2469
2470 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
2471 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
2472 wake_up_interruptible(&hba->ufshcd_tm_wait_queue);
2473}
2474
2475
2476
2477
2478
2479
2480static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
2481{
2482 hba->errors = UFSHCD_ERROR_MASK & intr_status;
2483 if (hba->errors)
2484 ufshcd_err_handler(hba);
2485
2486 if (intr_status & UFSHCD_UIC_MASK)
2487 ufshcd_uic_cmd_compl(hba, intr_status);
2488
2489 if (intr_status & UTP_TASK_REQ_COMPL)
2490 ufshcd_tmc_handler(hba);
2491
2492 if (intr_status & UTP_TRANSFER_REQ_COMPL)
2493 ufshcd_transfer_req_compl(hba);
2494}
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504static irqreturn_t ufshcd_intr(int irq, void *__hba)
2505{
2506 u32 intr_status;
2507 irqreturn_t retval = IRQ_NONE;
2508 struct ufs_hba *hba = __hba;
2509
2510 spin_lock(hba->host->host_lock);
2511 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
2512
2513 if (intr_status) {
2514 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
2515 ufshcd_sl_intr(hba, intr_status);
2516 retval = IRQ_HANDLED;
2517 }
2518 spin_unlock(hba->host->host_lock);
2519 return retval;
2520}
2521
2522
2523
2524
2525
2526
2527
2528
2529static int
2530ufshcd_issue_tm_cmd(struct ufs_hba *hba,
2531 struct ufshcd_lrb *lrbp,
2532 u8 tm_function)
2533{
2534 struct utp_task_req_desc *task_req_descp;
2535 struct utp_upiu_task_req *task_req_upiup;
2536 struct Scsi_Host *host;
2537 unsigned long flags;
2538 int free_slot = 0;
2539 int err;
2540
2541 host = hba->host;
2542
2543 spin_lock_irqsave(host->host_lock, flags);
2544
2545
2546 free_slot = ufshcd_get_tm_free_slot(hba);
2547 if (free_slot >= hba->nutmrs) {
2548 spin_unlock_irqrestore(host->host_lock, flags);
2549 dev_err(hba->dev, "Task management queue full\n");
2550 err = FAILED;
2551 goto out;
2552 }
2553
2554 task_req_descp = hba->utmrdl_base_addr;
2555 task_req_descp += free_slot;
2556
2557
2558 task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
2559 task_req_descp->header.dword_2 =
2560 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2561
2562
2563 task_req_upiup =
2564 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
2565 task_req_upiup->header.dword_0 =
2566 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
2567 lrbp->lun, lrbp->task_tag);
2568 task_req_upiup->header.dword_1 =
2569 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
2570
2571 task_req_upiup->input_param1 = lrbp->lun;
2572 task_req_upiup->input_param1 =
2573 cpu_to_be32(task_req_upiup->input_param1);
2574 task_req_upiup->input_param2 = lrbp->task_tag;
2575 task_req_upiup->input_param2 =
2576 cpu_to_be32(task_req_upiup->input_param2);
2577
2578
2579 __set_bit(free_slot, &hba->outstanding_tasks);
2580 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
2581
2582 spin_unlock_irqrestore(host->host_lock, flags);
2583
2584
2585 err =
2586 wait_event_interruptible_timeout(hba->ufshcd_tm_wait_queue,
2587 (test_bit(free_slot,
2588 &hba->tm_condition) != 0),
2589 60 * HZ);
2590 if (!err) {
2591 dev_err(hba->dev,
2592 "Task management command timed-out\n");
2593 err = FAILED;
2594 goto out;
2595 }
2596 clear_bit(free_slot, &hba->tm_condition);
2597 err = ufshcd_task_req_compl(hba, free_slot);
2598out:
2599 return err;
2600}
2601
2602
2603
2604
2605
2606
2607
2608static int ufshcd_device_reset(struct scsi_cmnd *cmd)
2609{
2610 struct Scsi_Host *host;
2611 struct ufs_hba *hba;
2612 unsigned int tag;
2613 u32 pos;
2614 int err;
2615
2616 host = cmd->device->host;
2617 hba = shost_priv(host);
2618 tag = cmd->request->tag;
2619
2620 err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_LOGICAL_RESET);
2621 if (err == FAILED)
2622 goto out;
2623
2624 for (pos = 0; pos < hba->nutrs; pos++) {
2625 if (test_bit(pos, &hba->outstanding_reqs) &&
2626 (hba->lrb[tag].lun == hba->lrb[pos].lun)) {
2627
2628
2629 ufshcd_utrl_clear(hba, pos);
2630
2631 clear_bit(pos, &hba->outstanding_reqs);
2632
2633 if (hba->lrb[pos].cmd) {
2634 scsi_dma_unmap(hba->lrb[pos].cmd);
2635 hba->lrb[pos].cmd->result =
2636 DID_ABORT << 16;
2637 hba->lrb[pos].cmd->scsi_done(cmd);
2638 hba->lrb[pos].cmd = NULL;
2639 clear_bit_unlock(pos, &hba->lrb_in_use);
2640 wake_up(&hba->dev_cmd.tag_wq);
2641 }
2642 }
2643 }
2644out:
2645 return err;
2646}
2647
2648
2649
2650
2651
2652
2653
2654static int ufshcd_host_reset(struct scsi_cmnd *cmd)
2655{
2656 struct ufs_hba *hba;
2657
2658 hba = shost_priv(cmd->device->host);
2659
2660 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
2661 return SUCCESS;
2662
2663 return ufshcd_do_reset(hba);
2664}
2665
2666
2667
2668
2669
2670
2671
2672static int ufshcd_abort(struct scsi_cmnd *cmd)
2673{
2674 struct Scsi_Host *host;
2675 struct ufs_hba *hba;
2676 unsigned long flags;
2677 unsigned int tag;
2678 int err;
2679
2680 host = cmd->device->host;
2681 hba = shost_priv(host);
2682 tag = cmd->request->tag;
2683
2684 spin_lock_irqsave(host->host_lock, flags);
2685
2686
2687 if (!(test_bit(tag, &hba->outstanding_reqs))) {
2688 err = FAILED;
2689 spin_unlock_irqrestore(host->host_lock, flags);
2690 goto out;
2691 }
2692 spin_unlock_irqrestore(host->host_lock, flags);
2693
2694 err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_ABORT_TASK);
2695 if (err == FAILED)
2696 goto out;
2697
2698 scsi_dma_unmap(cmd);
2699
2700 spin_lock_irqsave(host->host_lock, flags);
2701
2702
2703 ufshcd_utrl_clear(hba, tag);
2704
2705 __clear_bit(tag, &hba->outstanding_reqs);
2706 hba->lrb[tag].cmd = NULL;
2707 spin_unlock_irqrestore(host->host_lock, flags);
2708
2709 clear_bit_unlock(tag, &hba->lrb_in_use);
2710 wake_up(&hba->dev_cmd.tag_wq);
2711out:
2712 return err;
2713}
2714
2715
2716
2717
2718
2719
2720static void ufshcd_async_scan(void *data, async_cookie_t cookie)
2721{
2722 struct ufs_hba *hba = (struct ufs_hba *)data;
2723 int ret;
2724
2725 ret = ufshcd_link_startup(hba);
2726 if (ret)
2727 goto out;
2728
2729 ufshcd_config_max_pwr_mode(hba);
2730
2731 ret = ufshcd_verify_dev_init(hba);
2732 if (ret)
2733 goto out;
2734
2735 ret = ufshcd_complete_dev_init(hba);
2736 if (ret)
2737 goto out;
2738
2739 ufshcd_force_reset_auto_bkops(hba);
2740 scsi_scan_host(hba->host);
2741 pm_runtime_put_sync(hba->dev);
2742out:
2743 return;
2744}
2745
2746static struct scsi_host_template ufshcd_driver_template = {
2747 .module = THIS_MODULE,
2748 .name = UFSHCD,
2749 .proc_name = UFSHCD,
2750 .queuecommand = ufshcd_queuecommand,
2751 .slave_alloc = ufshcd_slave_alloc,
2752 .slave_destroy = ufshcd_slave_destroy,
2753 .eh_abort_handler = ufshcd_abort,
2754 .eh_device_reset_handler = ufshcd_device_reset,
2755 .eh_host_reset_handler = ufshcd_host_reset,
2756 .this_id = -1,
2757 .sg_tablesize = SG_ALL,
2758 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
2759 .can_queue = UFSHCD_CAN_QUEUE,
2760};
2761
2762
2763
2764
2765
2766
2767
2768
2769int ufshcd_suspend(struct ufs_hba *hba, pm_message_t state)
2770{
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780 return -ENOSYS;
2781}
2782EXPORT_SYMBOL_GPL(ufshcd_suspend);
2783
2784
2785
2786
2787
2788
2789
2790int ufshcd_resume(struct ufs_hba *hba)
2791{
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801 return -ENOSYS;
2802}
2803EXPORT_SYMBOL_GPL(ufshcd_resume);
2804
2805int ufshcd_runtime_suspend(struct ufs_hba *hba)
2806{
2807 if (!hba)
2808 return 0;
2809
2810
2811
2812
2813
2814 return ufshcd_enable_auto_bkops(hba);
2815}
2816EXPORT_SYMBOL(ufshcd_runtime_suspend);
2817
2818int ufshcd_runtime_resume(struct ufs_hba *hba)
2819{
2820 if (!hba)
2821 return 0;
2822
2823 return ufshcd_disable_auto_bkops(hba);
2824}
2825EXPORT_SYMBOL(ufshcd_runtime_resume);
2826
2827int ufshcd_runtime_idle(struct ufs_hba *hba)
2828{
2829 return 0;
2830}
2831EXPORT_SYMBOL(ufshcd_runtime_idle);
2832
2833
2834
2835
2836
2837
2838void ufshcd_remove(struct ufs_hba *hba)
2839{
2840 scsi_remove_host(hba->host);
2841
2842 ufshcd_disable_intr(hba, hba->intr_mask);
2843 ufshcd_hba_stop(hba);
2844
2845 scsi_host_put(hba->host);
2846}
2847EXPORT_SYMBOL_GPL(ufshcd_remove);
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857int ufshcd_init(struct device *dev, struct ufs_hba **hba_handle,
2858 void __iomem *mmio_base, unsigned int irq)
2859{
2860 struct Scsi_Host *host;
2861 struct ufs_hba *hba;
2862 int err;
2863
2864 if (!dev) {
2865 dev_err(dev,
2866 "Invalid memory reference for dev is NULL\n");
2867 err = -ENODEV;
2868 goto out_error;
2869 }
2870
2871 if (!mmio_base) {
2872 dev_err(dev,
2873 "Invalid memory reference for mmio_base is NULL\n");
2874 err = -ENODEV;
2875 goto out_error;
2876 }
2877
2878 host = scsi_host_alloc(&ufshcd_driver_template,
2879 sizeof(struct ufs_hba));
2880 if (!host) {
2881 dev_err(dev, "scsi_host_alloc failed\n");
2882 err = -ENOMEM;
2883 goto out_error;
2884 }
2885 hba = shost_priv(host);
2886 hba->host = host;
2887 hba->dev = dev;
2888 hba->mmio_base = mmio_base;
2889 hba->irq = irq;
2890
2891
2892 ufshcd_hba_capabilities(hba);
2893
2894
2895 hba->ufs_version = ufshcd_get_ufs_version(hba);
2896
2897
2898 hba->intr_mask = ufshcd_get_intr_mask(hba);
2899
2900
2901 err = ufshcd_memory_alloc(hba);
2902 if (err) {
2903 dev_err(hba->dev, "Memory allocation failed\n");
2904 goto out_disable;
2905 }
2906
2907
2908 ufshcd_host_memory_configure(hba);
2909
2910 host->can_queue = hba->nutrs;
2911 host->cmd_per_lun = hba->nutrs;
2912 host->max_id = UFSHCD_MAX_ID;
2913 host->max_lun = UFSHCD_MAX_LUNS;
2914 host->max_channel = UFSHCD_MAX_CHANNEL;
2915 host->unique_id = host->host_no;
2916 host->max_cmd_len = MAX_CDB_SIZE;
2917
2918
2919 init_waitqueue_head(&hba->ufshcd_tm_wait_queue);
2920
2921
2922 INIT_WORK(&hba->feh_workq, ufshcd_fatal_err_handler);
2923 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
2924
2925
2926 mutex_init(&hba->uic_cmd_mutex);
2927
2928
2929 mutex_init(&hba->dev_cmd.lock);
2930
2931
2932 init_waitqueue_head(&hba->dev_cmd.tag_wq);
2933
2934
2935 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
2936 if (err) {
2937 dev_err(hba->dev, "request irq failed\n");
2938 goto out_disable;
2939 }
2940
2941
2942 err = scsi_init_shared_tag_map(host, host->can_queue);
2943 if (err) {
2944 dev_err(hba->dev, "init shared queue failed\n");
2945 goto out_disable;
2946 }
2947
2948 err = scsi_add_host(host, hba->dev);
2949 if (err) {
2950 dev_err(hba->dev, "scsi_add_host failed\n");
2951 goto out_disable;
2952 }
2953
2954
2955 err = ufshcd_hba_enable(hba);
2956 if (err) {
2957 dev_err(hba->dev, "Host controller enable failed\n");
2958 goto out_remove_scsi_host;
2959 }
2960
2961 *hba_handle = hba;
2962
2963
2964 pm_runtime_get_sync(dev);
2965
2966 async_schedule(ufshcd_async_scan, hba);
2967
2968 return 0;
2969
2970out_remove_scsi_host:
2971 scsi_remove_host(hba->host);
2972out_disable:
2973 scsi_host_put(host);
2974out_error:
2975 return err;
2976}
2977EXPORT_SYMBOL_GPL(ufshcd_init);
2978
2979MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
2980MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
2981MODULE_DESCRIPTION("Generic UFS host controller driver Core");
2982MODULE_LICENSE("GPL");
2983MODULE_VERSION(UFSHCD_DRIVER_VERSION);
2984