1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/delay.h>
16#include <linux/list.h>
17#include <linux/module.h>
18#include <linux/pci.h>
19
20#include <asm/unaligned.h>
21
22#include <misc/cxl.h>
23
24#include <scsi/scsi_cmnd.h>
25#include <scsi/scsi_host.h>
26#include <uapi/scsi/cxlflash_ioctl.h>
27
28#include "main.h"
29#include "sislite.h"
30#include "common.h"
31
32MODULE_DESCRIPTION(CXLFLASH_ADAPTER_NAME);
33MODULE_AUTHOR("Manoj N. Kumar <manoj@linux.vnet.ibm.com>");
34MODULE_AUTHOR("Matthew R. Ochs <mrochs@linux.vnet.ibm.com>");
35MODULE_LICENSE("GPL");
36
37
38
39
40
41
42
43
44
45
46
47
48
49static struct afu_cmd *cmd_checkout(struct afu *afu)
50{
51 int k, dec = CXLFLASH_NUM_CMDS;
52 struct afu_cmd *cmd;
53
54 while (dec--) {
55 k = (afu->cmd_couts++ & (CXLFLASH_NUM_CMDS - 1));
56
57 cmd = &afu->cmd[k];
58
59 if (!atomic_dec_if_positive(&cmd->free)) {
60 pr_devel("%s: returning found index=%d cmd=%p\n",
61 __func__, cmd->slot, cmd);
62 memset(cmd->buf, 0, CMD_BUFSIZE);
63 memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb));
64 return cmd;
65 }
66 }
67
68 return NULL;
69}
70
71
72
73
74
75
76
77
78
79
80
81static void cmd_checkin(struct afu_cmd *cmd)
82{
83 cmd->rcb.scp = NULL;
84 cmd->rcb.timeout = 0;
85 cmd->sa.ioasc = 0;
86 cmd->cmd_tmf = false;
87 cmd->sa.host_use[0] = 0;
88
89 if (unlikely(atomic_inc_return(&cmd->free) != 1)) {
90 pr_err("%s: Freeing cmd (%d) that is not in use!\n",
91 __func__, cmd->slot);
92 return;
93 }
94
95 pr_devel("%s: released cmd %p index=%d\n", __func__, cmd, cmd->slot);
96}
97
98
99
100
101
102
103
104
105static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp)
106{
107 struct sisl_ioarcb *ioarcb;
108 struct sisl_ioasa *ioasa;
109 u32 resid;
110
111 if (unlikely(!cmd))
112 return;
113
114 ioarcb = &(cmd->rcb);
115 ioasa = &(cmd->sa);
116
117 if (ioasa->rc.flags & SISL_RC_FLAGS_UNDERRUN) {
118 resid = ioasa->resid;
119 scsi_set_resid(scp, resid);
120 pr_debug("%s: cmd underrun cmd = %p scp = %p, resid = %d\n",
121 __func__, cmd, scp, resid);
122 }
123
124 if (ioasa->rc.flags & SISL_RC_FLAGS_OVERRUN) {
125 pr_debug("%s: cmd underrun cmd = %p scp = %p\n",
126 __func__, cmd, scp);
127 scp->result = (DID_ERROR << 16);
128 }
129
130 pr_debug("%s: cmd failed afu_rc=%d scsi_rc=%d fc_rc=%d "
131 "afu_extra=0x%X, scsi_extra=0x%X, fc_extra=0x%X\n",
132 __func__, ioasa->rc.afu_rc, ioasa->rc.scsi_rc,
133 ioasa->rc.fc_rc, ioasa->afu_extra, ioasa->scsi_extra,
134 ioasa->fc_extra);
135
136 if (ioasa->rc.scsi_rc) {
137
138 if (ioasa->rc.flags & SISL_RC_FLAGS_SENSE_VALID) {
139 memcpy(scp->sense_buffer, ioasa->sense_data,
140 SISL_SENSE_DATA_LEN);
141 scp->result = ioasa->rc.scsi_rc;
142 } else
143 scp->result = ioasa->rc.scsi_rc | (DID_ERROR << 16);
144 }
145
146
147
148
149
150 if (ioasa->rc.fc_rc) {
151
152 switch (ioasa->rc.fc_rc) {
153 case SISL_FC_RC_LINKDOWN:
154 scp->result = (DID_REQUEUE << 16);
155 break;
156 case SISL_FC_RC_RESID:
157
158 if (!(ioasa->rc.flags & SISL_RC_FLAGS_OVERRUN)) {
159
160
161
162
163
164 scp->result = (DID_ERROR << 16);
165 }
166 break;
167 case SISL_FC_RC_RESIDERR:
168
169 case SISL_FC_RC_TGTABORT:
170 case SISL_FC_RC_ABORTOK:
171 case SISL_FC_RC_ABORTFAIL:
172 case SISL_FC_RC_NOLOGI:
173 case SISL_FC_RC_ABORTPEND:
174 case SISL_FC_RC_WRABORTPEND:
175 case SISL_FC_RC_NOEXP:
176 case SISL_FC_RC_INUSE:
177 scp->result = (DID_ERROR << 16);
178 break;
179 }
180 }
181
182 if (ioasa->rc.afu_rc) {
183
184 switch (ioasa->rc.afu_rc) {
185 case SISL_AFU_RC_NO_CHANNELS:
186 scp->result = (DID_NO_CONNECT << 16);
187 break;
188 case SISL_AFU_RC_DATA_DMA_ERR:
189 switch (ioasa->afu_extra) {
190 case SISL_AFU_DMA_ERR_PAGE_IN:
191
192 scp->result = (DID_IMM_RETRY << 16);
193 break;
194 case SISL_AFU_DMA_ERR_INVALID_EA:
195 default:
196 scp->result = (DID_ERROR << 16);
197 }
198 break;
199 case SISL_AFU_RC_OUT_OF_DATA_BUFS:
200
201 scp->result = (DID_ALLOC_FAILURE << 16);
202 break;
203 default:
204 scp->result = (DID_ERROR << 16);
205 }
206 }
207}
208
209
210
211
212
213
214
215
216
217static void cmd_complete(struct afu_cmd *cmd)
218{
219 struct scsi_cmnd *scp;
220 ulong lock_flags;
221 struct afu *afu = cmd->parent;
222 struct cxlflash_cfg *cfg = afu->parent;
223 bool cmd_is_tmf;
224
225 spin_lock_irqsave(&cmd->slock, lock_flags);
226 cmd->sa.host_use_b[0] |= B_DONE;
227 spin_unlock_irqrestore(&cmd->slock, lock_flags);
228
229 if (cmd->rcb.scp) {
230 scp = cmd->rcb.scp;
231 if (unlikely(cmd->sa.ioasc))
232 process_cmd_err(cmd, scp);
233 else
234 scp->result = (DID_OK << 16);
235
236 cmd_is_tmf = cmd->cmd_tmf;
237 cmd_checkin(cmd);
238
239 pr_debug_ratelimited("%s: calling scsi_done scp=%p result=%X "
240 "ioasc=%d\n", __func__, scp, scp->result,
241 cmd->sa.ioasc);
242
243 scsi_dma_unmap(scp);
244 scp->scsi_done(scp);
245
246 if (cmd_is_tmf) {
247 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
248 cfg->tmf_active = false;
249 wake_up_all_locked(&cfg->tmf_waitq);
250 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
251 }
252 } else
253 complete(&cmd->cevent);
254}
255
256
257
258
259
260
261
262static void context_reset(struct afu_cmd *cmd)
263{
264 int nretry = 0;
265 u64 rrin = 0x1;
266 u64 room = 0;
267 struct afu *afu = cmd->parent;
268 ulong lock_flags;
269
270 pr_debug("%s: cmd=%p\n", __func__, cmd);
271
272 spin_lock_irqsave(&cmd->slock, lock_flags);
273
274
275 if (cmd->sa.host_use_b[0] & B_DONE) {
276 spin_unlock_irqrestore(&cmd->slock, lock_flags);
277 return;
278 }
279
280 cmd->sa.host_use_b[0] |= (B_DONE | B_ERROR | B_TIMEOUT);
281 spin_unlock_irqrestore(&cmd->slock, lock_flags);
282
283
284
285
286
287 do {
288 room = readq_be(&afu->host_map->cmd_room);
289 atomic64_set(&afu->room, room);
290 if (room)
291 goto write_rrin;
292 udelay(nretry);
293 } while (nretry++ < MC_ROOM_RETRY_CNT);
294
295 pr_err("%s: no cmd_room to send reset\n", __func__);
296 return;
297
298write_rrin:
299 nretry = 0;
300 writeq_be(rrin, &afu->host_map->ioarrin);
301 do {
302 rrin = readq_be(&afu->host_map->ioarrin);
303 if (rrin != 0x1)
304 break;
305
306 udelay(2 << nretry);
307 } while (nretry++ < MC_ROOM_RETRY_CNT);
308}
309
310
311
312
313
314
315
316
317
318static int send_cmd(struct afu *afu, struct afu_cmd *cmd)
319{
320 struct cxlflash_cfg *cfg = afu->parent;
321 struct device *dev = &cfg->dev->dev;
322 int nretry = 0;
323 int rc = 0;
324 u64 room;
325 long newval;
326
327
328
329
330
331
332
333retry:
334 newval = atomic64_dec_if_positive(&afu->room);
335 if (!newval) {
336 do {
337 room = readq_be(&afu->host_map->cmd_room);
338 atomic64_set(&afu->room, room);
339 if (room)
340 goto write_ioarrin;
341 udelay(nretry);
342 } while (nretry++ < MC_ROOM_RETRY_CNT);
343
344 dev_err(dev, "%s: no cmd_room to send 0x%X\n",
345 __func__, cmd->rcb.cdb[0]);
346
347 goto no_room;
348 } else if (unlikely(newval < 0)) {
349
350
351
352
353
354 if (nretry++ < MC_ROOM_RETRY_CNT) {
355 udelay(nretry);
356 goto retry;
357 }
358
359 goto no_room;
360 }
361
362write_ioarrin:
363 writeq_be((u64)&cmd->rcb, &afu->host_map->ioarrin);
364out:
365 pr_devel("%s: cmd=%p len=%d ea=%p rc=%d\n", __func__, cmd,
366 cmd->rcb.data_len, (void *)cmd->rcb.data_ea, rc);
367 return rc;
368
369no_room:
370 afu->read_room = true;
371 schedule_work(&cfg->work_q);
372 rc = SCSI_MLQUEUE_HOST_BUSY;
373 goto out;
374}
375
376
377
378
379
380
381static void wait_resp(struct afu *afu, struct afu_cmd *cmd)
382{
383 ulong timeout = msecs_to_jiffies(cmd->rcb.timeout * 2 * 1000);
384
385 timeout = wait_for_completion_timeout(&cmd->cevent, timeout);
386 if (!timeout)
387 context_reset(cmd);
388
389 if (unlikely(cmd->sa.ioasc != 0))
390 pr_err("%s: CMD 0x%X failed, IOASC: flags 0x%X, afu_rc 0x%X, "
391 "scsi_rc 0x%X, fc_rc 0x%X\n", __func__, cmd->rcb.cdb[0],
392 cmd->sa.rc.flags, cmd->sa.rc.afu_rc, cmd->sa.rc.scsi_rc,
393 cmd->sa.rc.fc_rc);
394}
395
396
397
398
399
400
401
402
403
404
405static int send_tmf(struct afu *afu, struct scsi_cmnd *scp, u64 tmfcmd)
406{
407 struct afu_cmd *cmd;
408
409 u32 port_sel = scp->device->channel + 1;
410 short lflag = 0;
411 struct Scsi_Host *host = scp->device->host;
412 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
413 struct device *dev = &cfg->dev->dev;
414 ulong lock_flags;
415 int rc = 0;
416 ulong to;
417
418 cmd = cmd_checkout(afu);
419 if (unlikely(!cmd)) {
420 dev_err(dev, "%s: could not get a free command\n", __func__);
421 rc = SCSI_MLQUEUE_HOST_BUSY;
422 goto out;
423 }
424
425
426 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
427 if (cfg->tmf_active)
428 wait_event_interruptible_lock_irq(cfg->tmf_waitq,
429 !cfg->tmf_active,
430 cfg->tmf_slock);
431 cfg->tmf_active = true;
432 cmd->cmd_tmf = true;
433 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
434
435 cmd->rcb.ctx_id = afu->ctx_hndl;
436 cmd->rcb.port_sel = port_sel;
437 cmd->rcb.lun_id = lun_to_lunid(scp->device->lun);
438
439 lflag = SISL_REQ_FLAGS_TMF_CMD;
440
441 cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID |
442 SISL_REQ_FLAGS_SUP_UNDERRUN | lflag);
443
444
445 cmd->rcb.scp = scp;
446
447
448 memcpy(cmd->rcb.cdb, &tmfcmd, sizeof(tmfcmd));
449
450
451 rc = send_cmd(afu, cmd);
452 if (unlikely(rc)) {
453 cmd_checkin(cmd);
454 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
455 cfg->tmf_active = false;
456 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
457 goto out;
458 }
459
460 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
461 to = msecs_to_jiffies(5000);
462 to = wait_event_interruptible_lock_irq_timeout(cfg->tmf_waitq,
463 !cfg->tmf_active,
464 cfg->tmf_slock,
465 to);
466 if (!to) {
467 cfg->tmf_active = false;
468 dev_err(dev, "%s: TMF timed out!\n", __func__);
469 rc = -1;
470 }
471 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
472out:
473 return rc;
474}
475
476
477
478
479
480
481
482static const char *cxlflash_driver_info(struct Scsi_Host *host)
483{
484 return CXLFLASH_ADAPTER_NAME;
485}
486
487
488
489
490
491
492
493
494static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
495{
496 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
497 struct afu *afu = cfg->afu;
498 struct device *dev = &cfg->dev->dev;
499 struct afu_cmd *cmd;
500 u32 port_sel = scp->device->channel + 1;
501 int nseg, i, ncount;
502 struct scatterlist *sg;
503 ulong lock_flags;
504 short lflag = 0;
505 int rc = 0;
506
507 dev_dbg_ratelimited(dev, "%s: (scp=%p) %d/%d/%d/%llu "
508 "cdb=(%08X-%08X-%08X-%08X)\n",
509 __func__, scp, host->host_no, scp->device->channel,
510 scp->device->id, scp->device->lun,
511 get_unaligned_be32(&((u32 *)scp->cmnd)[0]),
512 get_unaligned_be32(&((u32 *)scp->cmnd)[1]),
513 get_unaligned_be32(&((u32 *)scp->cmnd)[2]),
514 get_unaligned_be32(&((u32 *)scp->cmnd)[3]));
515
516
517
518
519
520 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
521 if (cfg->tmf_active) {
522 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
523 rc = SCSI_MLQUEUE_HOST_BUSY;
524 goto out;
525 }
526 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
527
528 switch (cfg->state) {
529 case STATE_RESET:
530 dev_dbg_ratelimited(dev, "%s: device is in reset!\n", __func__);
531 rc = SCSI_MLQUEUE_HOST_BUSY;
532 goto out;
533 case STATE_FAILTERM:
534 dev_dbg_ratelimited(dev, "%s: device has failed!\n", __func__);
535 scp->result = (DID_NO_CONNECT << 16);
536 scp->scsi_done(scp);
537 rc = 0;
538 goto out;
539 default:
540 break;
541 }
542
543 cmd = cmd_checkout(afu);
544 if (unlikely(!cmd)) {
545 dev_err(dev, "%s: could not get a free command\n", __func__);
546 rc = SCSI_MLQUEUE_HOST_BUSY;
547 goto out;
548 }
549
550 cmd->rcb.ctx_id = afu->ctx_hndl;
551 cmd->rcb.port_sel = port_sel;
552 cmd->rcb.lun_id = lun_to_lunid(scp->device->lun);
553
554 if (scp->sc_data_direction == DMA_TO_DEVICE)
555 lflag = SISL_REQ_FLAGS_HOST_WRITE;
556 else
557 lflag = SISL_REQ_FLAGS_HOST_READ;
558
559 cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID |
560 SISL_REQ_FLAGS_SUP_UNDERRUN | lflag);
561
562
563 cmd->rcb.scp = scp;
564
565 nseg = scsi_dma_map(scp);
566 if (unlikely(nseg < 0)) {
567 dev_err(dev, "%s: Fail DMA map! nseg=%d\n",
568 __func__, nseg);
569 rc = SCSI_MLQUEUE_HOST_BUSY;
570 goto out;
571 }
572
573 ncount = scsi_sg_count(scp);
574 scsi_for_each_sg(scp, sg, ncount, i) {
575 cmd->rcb.data_len = sg_dma_len(sg);
576 cmd->rcb.data_ea = sg_dma_address(sg);
577 }
578
579
580 memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb));
581
582
583 rc = send_cmd(afu, cmd);
584 if (unlikely(rc)) {
585 cmd_checkin(cmd);
586 scsi_dma_unmap(scp);
587 }
588
589out:
590 pr_devel("%s: returning rc=%d\n", __func__, rc);
591 return rc;
592}
593
594
595
596
597
598static void cxlflash_wait_for_pci_err_recovery(struct cxlflash_cfg *cfg)
599{
600 struct pci_dev *pdev = cfg->dev;
601
602 if (pci_channel_offline(pdev))
603 wait_event_timeout(cfg->reset_waitq,
604 !pci_channel_offline(pdev),
605 CXLFLASH_PCI_ERROR_RECOVERY_TIMEOUT);
606}
607
608
609
610
611
612static void free_mem(struct cxlflash_cfg *cfg)
613{
614 int i;
615 char *buf = NULL;
616 struct afu *afu = cfg->afu;
617
618 if (cfg->afu) {
619 for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
620 buf = afu->cmd[i].buf;
621 if (!((u64)buf & (PAGE_SIZE - 1)))
622 free_page((ulong)buf);
623 }
624
625 free_pages((ulong)afu, get_order(sizeof(struct afu)));
626 cfg->afu = NULL;
627 }
628}
629
630
631
632
633
634
635
636static void stop_afu(struct cxlflash_cfg *cfg)
637{
638 int i;
639 struct afu *afu = cfg->afu;
640
641 if (likely(afu)) {
642 for (i = 0; i < CXLFLASH_NUM_CMDS; i++)
643 complete(&afu->cmd[i].cevent);
644
645 if (likely(afu->afu_map)) {
646 cxl_psa_unmap((void __iomem *)afu->afu_map);
647 afu->afu_map = NULL;
648 }
649 }
650}
651
652
653
654
655
656
657
658
659static void term_mc(struct cxlflash_cfg *cfg, enum undo_level level)
660{
661 int rc = 0;
662 struct afu *afu = cfg->afu;
663 struct device *dev = &cfg->dev->dev;
664
665 if (!afu || !cfg->mcctx) {
666 dev_err(dev, "%s: returning from term_mc with NULL afu or MC\n",
667 __func__);
668 return;
669 }
670
671 switch (level) {
672 case UNDO_START:
673 rc = cxl_stop_context(cfg->mcctx);
674 BUG_ON(rc);
675 case UNMAP_THREE:
676 cxl_unmap_afu_irq(cfg->mcctx, 3, afu);
677 case UNMAP_TWO:
678 cxl_unmap_afu_irq(cfg->mcctx, 2, afu);
679 case UNMAP_ONE:
680 cxl_unmap_afu_irq(cfg->mcctx, 1, afu);
681 case FREE_IRQ:
682 cxl_free_afu_irqs(cfg->mcctx);
683 case RELEASE_CONTEXT:
684 cfg->mcctx = NULL;
685 }
686}
687
688
689
690
691
692
693
694static void term_afu(struct cxlflash_cfg *cfg)
695{
696 term_mc(cfg, UNDO_START);
697
698 if (cfg->afu)
699 stop_afu(cfg);
700
701 pr_debug("%s: returning\n", __func__);
702}
703
704
705
706
707
708
709
710static void cxlflash_remove(struct pci_dev *pdev)
711{
712 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
713 ulong lock_flags;
714
715
716
717
718 spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
719 if (cfg->tmf_active)
720 wait_event_interruptible_lock_irq(cfg->tmf_waitq,
721 !cfg->tmf_active,
722 cfg->tmf_slock);
723 spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
724
725 cfg->state = STATE_FAILTERM;
726 cxlflash_stop_term_user_contexts(cfg);
727
728 switch (cfg->init_state) {
729 case INIT_STATE_SCSI:
730 cxlflash_term_local_luns(cfg);
731 scsi_remove_host(cfg->host);
732
733 case INIT_STATE_AFU:
734 term_afu(cfg);
735 cancel_work_sync(&cfg->work_q);
736 case INIT_STATE_PCI:
737 pci_release_regions(cfg->dev);
738 pci_disable_device(pdev);
739 case INIT_STATE_NONE:
740 free_mem(cfg);
741 scsi_host_put(cfg->host);
742 break;
743 }
744
745 pr_debug("%s: returning\n", __func__);
746}
747
748
749
750
751
752
753
754
755
756
757
758static int alloc_mem(struct cxlflash_cfg *cfg)
759{
760 int rc = 0;
761 int i;
762 char *buf = NULL;
763 struct device *dev = &cfg->dev->dev;
764
765
766 cfg->afu = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
767 get_order(sizeof(struct afu)));
768 if (unlikely(!cfg->afu)) {
769 dev_err(dev, "%s: cannot get %d free pages\n",
770 __func__, get_order(sizeof(struct afu)));
771 rc = -ENOMEM;
772 goto out;
773 }
774 cfg->afu->parent = cfg;
775 cfg->afu->afu_map = NULL;
776
777 for (i = 0; i < CXLFLASH_NUM_CMDS; buf += CMD_BUFSIZE, i++) {
778 if (!((u64)buf & (PAGE_SIZE - 1))) {
779 buf = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
780 if (unlikely(!buf)) {
781 dev_err(dev,
782 "%s: Allocate command buffers fail!\n",
783 __func__);
784 rc = -ENOMEM;
785 free_mem(cfg);
786 goto out;
787 }
788 }
789
790 cfg->afu->cmd[i].buf = buf;
791 atomic_set(&cfg->afu->cmd[i].free, 1);
792 cfg->afu->cmd[i].slot = i;
793 }
794
795out:
796 return rc;
797}
798
799
800
801
802
803
804
805static int init_pci(struct cxlflash_cfg *cfg)
806{
807 struct pci_dev *pdev = cfg->dev;
808 int rc = 0;
809
810 cfg->cxlflash_regs_pci = pci_resource_start(pdev, 0);
811 rc = pci_request_regions(pdev, CXLFLASH_NAME);
812 if (rc < 0) {
813 dev_err(&pdev->dev,
814 "%s: Couldn't register memory range of registers\n",
815 __func__);
816 goto out;
817 }
818
819 rc = pci_enable_device(pdev);
820 if (rc || pci_channel_offline(pdev)) {
821 if (pci_channel_offline(pdev)) {
822 cxlflash_wait_for_pci_err_recovery(cfg);
823 rc = pci_enable_device(pdev);
824 }
825
826 if (rc) {
827 dev_err(&pdev->dev, "%s: Cannot enable adapter\n",
828 __func__);
829 cxlflash_wait_for_pci_err_recovery(cfg);
830 goto out_release_regions;
831 }
832 }
833
834 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
835 if (rc < 0) {
836 dev_dbg(&pdev->dev, "%s: Failed to set 64 bit PCI DMA mask\n",
837 __func__);
838 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
839 }
840
841 if (rc < 0) {
842 dev_err(&pdev->dev, "%s: Failed to set PCI DMA mask\n",
843 __func__);
844 goto out_disable;
845 }
846
847 pci_set_master(pdev);
848
849 if (pci_channel_offline(pdev)) {
850 cxlflash_wait_for_pci_err_recovery(cfg);
851 if (pci_channel_offline(pdev)) {
852 rc = -EIO;
853 goto out_msi_disable;
854 }
855 }
856
857 rc = pci_save_state(pdev);
858
859 if (rc != PCIBIOS_SUCCESSFUL) {
860 dev_err(&pdev->dev, "%s: Failed to save PCI config space\n",
861 __func__);
862 rc = -EIO;
863 goto cleanup_nolog;
864 }
865
866out:
867 pr_debug("%s: returning rc=%d\n", __func__, rc);
868 return rc;
869
870cleanup_nolog:
871out_msi_disable:
872 cxlflash_wait_for_pci_err_recovery(cfg);
873out_disable:
874 pci_disable_device(pdev);
875out_release_regions:
876 pci_release_regions(pdev);
877 goto out;
878
879}
880
881
882
883
884
885
886
887static int init_scsi(struct cxlflash_cfg *cfg)
888{
889 struct pci_dev *pdev = cfg->dev;
890 int rc = 0;
891
892 rc = scsi_add_host(cfg->host, &pdev->dev);
893 if (rc) {
894 dev_err(&pdev->dev, "%s: scsi_add_host failed (rc=%d)\n",
895 __func__, rc);
896 goto out;
897 }
898
899 scsi_scan_host(cfg->host);
900
901out:
902 pr_debug("%s: returning rc=%d\n", __func__, rc);
903 return rc;
904}
905
906
907
908
909
910
911
912
913
914static void set_port_online(__be64 __iomem *fc_regs)
915{
916 u64 cmdcfg;
917
918 cmdcfg = readq_be(&fc_regs[FC_MTIP_CMDCONFIG / 8]);
919 cmdcfg &= (~FC_MTIP_CMDCONFIG_OFFLINE);
920 cmdcfg |= (FC_MTIP_CMDCONFIG_ONLINE);
921 writeq_be(cmdcfg, &fc_regs[FC_MTIP_CMDCONFIG / 8]);
922}
923
924
925
926
927
928
929
930static void set_port_offline(__be64 __iomem *fc_regs)
931{
932 u64 cmdcfg;
933
934 cmdcfg = readq_be(&fc_regs[FC_MTIP_CMDCONFIG / 8]);
935 cmdcfg &= (~FC_MTIP_CMDCONFIG_ONLINE);
936 cmdcfg |= (FC_MTIP_CMDCONFIG_OFFLINE);
937 writeq_be(cmdcfg, &fc_regs[FC_MTIP_CMDCONFIG / 8]);
938}
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954static int wait_port_online(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry)
955{
956 u64 status;
957
958 if (delay_us < 1000) {
959 pr_err("%s: invalid delay specified %d\n", __func__, delay_us);
960 return -EINVAL;
961 }
962
963 do {
964 msleep(delay_us / 1000);
965 status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]);
966 } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_ONLINE &&
967 nretry--);
968
969 return ((status & FC_MTIP_STATUS_MASK) == FC_MTIP_STATUS_ONLINE);
970}
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985static int wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry)
986{
987 u64 status;
988
989 if (delay_us < 1000) {
990 pr_err("%s: invalid delay specified %d\n", __func__, delay_us);
991 return -EINVAL;
992 }
993
994 do {
995 msleep(delay_us / 1000);
996 status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]);
997 } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_OFFLINE &&
998 nretry--);
999
1000 return ((status & FC_MTIP_STATUS_MASK) == FC_MTIP_STATUS_OFFLINE);
1001}
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020static int afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs,
1021 u64 wwpn)
1022{
1023 int rc = 0;
1024
1025 set_port_offline(fc_regs);
1026
1027 if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
1028 FC_PORT_STATUS_RETRY_CNT)) {
1029 pr_debug("%s: wait on port %d to go offline timed out\n",
1030 __func__, port);
1031 rc = -1;
1032 }
1033
1034 if (rc == 0)
1035 writeq_be(wwpn, &fc_regs[FC_PNAME / 8]);
1036
1037
1038 rc = 0;
1039
1040 set_port_online(fc_regs);
1041
1042 if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
1043 FC_PORT_STATUS_RETRY_CNT)) {
1044 pr_err("%s: wait on port %d to go online timed out\n",
1045 __func__, port);
1046 }
1047
1048 pr_debug("%s: returning rc=%d\n", __func__, rc);
1049
1050 return rc;
1051}
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066static void afu_link_reset(struct afu *afu, int port, __be64 __iomem *fc_regs)
1067{
1068 u64 port_sel;
1069
1070
1071 port_sel = readq_be(&afu->afu_map->global.regs.afu_port_sel);
1072 port_sel &= ~(1ULL << port);
1073 writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel);
1074 cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC);
1075
1076 set_port_offline(fc_regs);
1077 if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
1078 FC_PORT_STATUS_RETRY_CNT))
1079 pr_err("%s: wait on port %d to go offline timed out\n",
1080 __func__, port);
1081
1082 set_port_online(fc_regs);
1083 if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
1084 FC_PORT_STATUS_RETRY_CNT))
1085 pr_err("%s: wait on port %d to go online timed out\n",
1086 __func__, port);
1087
1088
1089 port_sel |= (1ULL << port);
1090 writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel);
1091 cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC);
1092
1093 pr_debug("%s: returning port_sel=%lld\n", __func__, port_sel);
1094}
1095
1096
1097
1098
1099static const struct asyc_intr_info ainfo[] = {
1100 {SISL_ASTATUS_FC0_OTHER, "other error", 0, CLR_FC_ERROR | LINK_RESET},
1101 {SISL_ASTATUS_FC0_LOGO, "target initiated LOGO", 0, 0},
1102 {SISL_ASTATUS_FC0_CRC_T, "CRC threshold exceeded", 0, LINK_RESET},
1103 {SISL_ASTATUS_FC0_LOGI_R, "login timed out, retrying", 0, LINK_RESET},
1104 {SISL_ASTATUS_FC0_LOGI_F, "login failed", 0, CLR_FC_ERROR},
1105 {SISL_ASTATUS_FC0_LOGI_S, "login succeeded", 0, SCAN_HOST},
1106 {SISL_ASTATUS_FC0_LINK_DN, "link down", 0, 0},
1107 {SISL_ASTATUS_FC0_LINK_UP, "link up", 0, SCAN_HOST},
1108 {SISL_ASTATUS_FC1_OTHER, "other error", 1, CLR_FC_ERROR | LINK_RESET},
1109 {SISL_ASTATUS_FC1_LOGO, "target initiated LOGO", 1, 0},
1110 {SISL_ASTATUS_FC1_CRC_T, "CRC threshold exceeded", 1, LINK_RESET},
1111 {SISL_ASTATUS_FC1_LOGI_R, "login timed out, retrying", 1, 0},
1112 {SISL_ASTATUS_FC1_LOGI_F, "login failed", 1, CLR_FC_ERROR},
1113 {SISL_ASTATUS_FC1_LOGI_S, "login succeeded", 1, SCAN_HOST},
1114 {SISL_ASTATUS_FC1_LINK_DN, "link down", 1, 0},
1115 {SISL_ASTATUS_FC1_LINK_UP, "link up", 1, SCAN_HOST},
1116 {0x0, "", 0, 0}
1117};
1118
1119
1120
1121
1122
1123
1124
1125static const struct asyc_intr_info *find_ainfo(u64 status)
1126{
1127 const struct asyc_intr_info *info;
1128
1129 for (info = &ainfo[0]; info->status; info++)
1130 if (info->status == status)
1131 return info;
1132
1133 return NULL;
1134}
1135
1136
1137
1138
1139
1140static void afu_err_intr_init(struct afu *afu)
1141{
1142 int i;
1143 u64 reg;
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_mask);
1154
1155 reg = ((u64) (((afu->ctx_hndl << 8) | SISL_MSI_ASYNC_ERROR)) << 40);
1156
1157 if (afu->internal_lun)
1158 reg |= 1;
1159 writeq_be(reg, &afu->afu_map->global.regs.afu_ctrl);
1160
1161 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear);
1162
1163
1164 writeq_be(SISL_ASTATUS_MASK, &afu->afu_map->global.regs.aintr_mask);
1165
1166
1167 writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear);
1168
1169
1170 reg = readq_be(&afu->afu_map->global.fc_regs[0][FC_CONFIG2 / 8]);
1171 reg &= SISL_FC_INTERNAL_MASK;
1172 if (afu->internal_lun)
1173 reg |= ((u64)(afu->internal_lun - 1) << SISL_FC_INTERNAL_SHIFT);
1174 writeq_be(reg, &afu->afu_map->global.fc_regs[0][FC_CONFIG2 / 8]);
1175
1176
1177 for (i = 0; i < NUM_FC_PORTS; i++) {
1178 writeq_be(0xFFFFFFFFU,
1179 &afu->afu_map->global.fc_regs[i][FC_ERROR / 8]);
1180 writeq_be(0, &afu->afu_map->global.fc_regs[i][FC_ERRCAP / 8]);
1181 }
1182
1183
1184
1185
1186
1187
1188
1189 writeq_be(SISL_MSI_SYNC_ERROR, &afu->host_map->ctx_ctrl);
1190 writeq_be(SISL_ISTATUS_MASK, &afu->host_map->intr_mask);
1191}
1192
1193
1194
1195
1196
1197
1198
1199
1200static irqreturn_t cxlflash_sync_err_irq(int irq, void *data)
1201{
1202 struct afu *afu = (struct afu *)data;
1203 u64 reg;
1204 u64 reg_unmasked;
1205
1206 reg = readq_be(&afu->host_map->intr_status);
1207 reg_unmasked = (reg & SISL_ISTATUS_UNMASK);
1208
1209 if (reg_unmasked == 0UL) {
1210 pr_err("%s: %llX: spurious interrupt, intr_status %016llX\n",
1211 __func__, (u64)afu, reg);
1212 goto cxlflash_sync_err_irq_exit;
1213 }
1214
1215 pr_err("%s: %llX: unexpected interrupt, intr_status %016llX\n",
1216 __func__, (u64)afu, reg);
1217
1218 writeq_be(reg_unmasked, &afu->host_map->intr_clear);
1219
1220cxlflash_sync_err_irq_exit:
1221 pr_debug("%s: returning rc=%d\n", __func__, IRQ_HANDLED);
1222 return IRQ_HANDLED;
1223}
1224
1225
1226
1227
1228
1229
1230
1231
1232static irqreturn_t cxlflash_rrq_irq(int irq, void *data)
1233{
1234 struct afu *afu = (struct afu *)data;
1235 struct afu_cmd *cmd;
1236 bool toggle = afu->toggle;
1237 u64 entry,
1238 *hrrq_start = afu->hrrq_start,
1239 *hrrq_end = afu->hrrq_end,
1240 *hrrq_curr = afu->hrrq_curr;
1241
1242
1243 while (true) {
1244 entry = *hrrq_curr;
1245
1246 if ((entry & SISL_RESP_HANDLE_T_BIT) != toggle)
1247 break;
1248
1249 cmd = (struct afu_cmd *)(entry & ~SISL_RESP_HANDLE_T_BIT);
1250 cmd_complete(cmd);
1251
1252
1253 if (hrrq_curr < hrrq_end)
1254 hrrq_curr++;
1255 else {
1256 hrrq_curr = hrrq_start;
1257 toggle ^= SISL_RESP_HANDLE_T_BIT;
1258 }
1259 }
1260
1261 afu->hrrq_curr = hrrq_curr;
1262 afu->toggle = toggle;
1263
1264 return IRQ_HANDLED;
1265}
1266
1267
1268
1269
1270
1271
1272
1273
1274static irqreturn_t cxlflash_async_err_irq(int irq, void *data)
1275{
1276 struct afu *afu = (struct afu *)data;
1277 struct cxlflash_cfg *cfg = afu->parent;
1278 struct device *dev = &cfg->dev->dev;
1279 u64 reg_unmasked;
1280 const struct asyc_intr_info *info;
1281 struct sisl_global_map __iomem *global = &afu->afu_map->global;
1282 u64 reg;
1283 u8 port;
1284 int i;
1285
1286 reg = readq_be(&global->regs.aintr_status);
1287 reg_unmasked = (reg & SISL_ASTATUS_UNMASK);
1288
1289 if (reg_unmasked == 0) {
1290 dev_err(dev, "%s: spurious interrupt, aintr_status 0x%016llX\n",
1291 __func__, reg);
1292 goto out;
1293 }
1294
1295
1296 writeq_be(reg_unmasked, &global->regs.aintr_clear);
1297
1298
1299 for (i = 0; reg_unmasked; i++, reg_unmasked = (reg_unmasked >> 1)) {
1300 info = find_ainfo(1ULL << i);
1301 if (((reg_unmasked & 0x1) == 0) || !info)
1302 continue;
1303
1304 port = info->port;
1305
1306 dev_err(dev, "%s: FC Port %d -> %s, fc_status 0x%08llX\n",
1307 __func__, port, info->desc,
1308 readq_be(&global->fc_regs[port][FC_STATUS / 8]));
1309
1310
1311
1312
1313
1314 if (info->action & LINK_RESET) {
1315 dev_err(dev, "%s: FC Port %d: resetting link\n",
1316 __func__, port);
1317 cfg->lr_state = LINK_RESET_REQUIRED;
1318 cfg->lr_port = port;
1319 schedule_work(&cfg->work_q);
1320 }
1321
1322 if (info->action & CLR_FC_ERROR) {
1323 reg = readq_be(&global->fc_regs[port][FC_ERROR / 8]);
1324
1325
1326
1327
1328
1329
1330 dev_err(dev, "%s: fc %d: clearing fc_error 0x%08llX\n",
1331 __func__, port, reg);
1332
1333 writeq_be(reg, &global->fc_regs[port][FC_ERROR / 8]);
1334 writeq_be(0, &global->fc_regs[port][FC_ERRCAP / 8]);
1335 }
1336
1337 if (info->action & SCAN_HOST) {
1338 atomic_inc(&cfg->scan_host_needed);
1339 schedule_work(&cfg->work_q);
1340 }
1341 }
1342
1343out:
1344 dev_dbg(dev, "%s: returning IRQ_HANDLED, afu=%p\n", __func__, afu);
1345 return IRQ_HANDLED;
1346}
1347
1348
1349
1350
1351
1352
1353
1354static int start_context(struct cxlflash_cfg *cfg)
1355{
1356 int rc = 0;
1357
1358 rc = cxl_start_context(cfg->mcctx,
1359 cfg->afu->work.work_element_descriptor,
1360 NULL);
1361
1362 pr_debug("%s: returning rc=%d\n", __func__, rc);
1363 return rc;
1364}
1365
1366
1367
1368
1369
1370
1371
1372
1373static int read_vpd(struct cxlflash_cfg *cfg, u64 wwpn[])
1374{
1375 struct pci_dev *dev = cfg->parent_dev;
1376 int rc = 0;
1377 int ro_start, ro_size, i, j, k;
1378 ssize_t vpd_size;
1379 char vpd_data[CXLFLASH_VPD_LEN];
1380 char tmp_buf[WWPN_BUF_LEN] = { 0 };
1381 char *wwpn_vpd_tags[NUM_FC_PORTS] = { "V5", "V6" };
1382
1383
1384 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
1385 if (unlikely(vpd_size <= 0)) {
1386 dev_err(&dev->dev, "%s: Unable to read VPD (size = %ld)\n",
1387 __func__, vpd_size);
1388 rc = -ENODEV;
1389 goto out;
1390 }
1391
1392
1393 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size,
1394 PCI_VPD_LRDT_RO_DATA);
1395 if (unlikely(ro_start < 0)) {
1396 dev_err(&dev->dev, "%s: VPD Read-only data not found\n",
1397 __func__);
1398 rc = -ENODEV;
1399 goto out;
1400 }
1401
1402
1403 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
1404 j = ro_size;
1405 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
1406 if (unlikely((i + j) > vpd_size)) {
1407 pr_debug("%s: Might need to read more VPD (%d > %ld)\n",
1408 __func__, (i + j), vpd_size);
1409 ro_size = vpd_size - i;
1410 }
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420 for (k = 0; k < NUM_FC_PORTS; k++) {
1421 j = ro_size;
1422 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
1423
1424 i = pci_vpd_find_info_keyword(vpd_data, i, j, wwpn_vpd_tags[k]);
1425 if (unlikely(i < 0)) {
1426 dev_err(&dev->dev, "%s: Port %d WWPN not found "
1427 "in VPD\n", __func__, k);
1428 rc = -ENODEV;
1429 goto out;
1430 }
1431
1432 j = pci_vpd_info_field_size(&vpd_data[i]);
1433 i += PCI_VPD_INFO_FLD_HDR_SIZE;
1434 if (unlikely((i + j > vpd_size) || (j != WWPN_LEN))) {
1435 dev_err(&dev->dev, "%s: Port %d WWPN incomplete or "
1436 "VPD corrupt\n",
1437 __func__, k);
1438 rc = -ENODEV;
1439 goto out;
1440 }
1441
1442 memcpy(tmp_buf, &vpd_data[i], WWPN_LEN);
1443 rc = kstrtoul(tmp_buf, WWPN_LEN, (ulong *)&wwpn[k]);
1444 if (unlikely(rc)) {
1445 dev_err(&dev->dev, "%s: Fail to convert port %d WWPN "
1446 "to integer\n", __func__, k);
1447 rc = -ENODEV;
1448 goto out;
1449 }
1450 }
1451
1452out:
1453 pr_debug("%s: returning rc=%d\n", __func__, rc);
1454 return rc;
1455}
1456
1457
1458
1459
1460
1461
1462
1463
1464static void init_pcr(struct cxlflash_cfg *cfg)
1465{
1466 struct afu *afu = cfg->afu;
1467 struct sisl_ctrl_map __iomem *ctrl_map;
1468 int i;
1469
1470 for (i = 0; i < MAX_CONTEXT; i++) {
1471 ctrl_map = &afu->afu_map->ctrls[i].ctrl;
1472
1473
1474 writeq_be(0, &ctrl_map->rht_start);
1475 writeq_be(0, &ctrl_map->rht_cnt_id);
1476 writeq_be(0, &ctrl_map->ctx_cap);
1477 }
1478
1479
1480 afu->ctx_hndl = (u16) cxl_process_element(cfg->mcctx);
1481 afu->host_map = &afu->afu_map->hosts[afu->ctx_hndl].host;
1482 afu->ctrl_map = &afu->afu_map->ctrls[afu->ctx_hndl].ctrl;
1483
1484
1485 writeq_be(SISL_ENDIAN_CTRL, &afu->host_map->endian_ctrl);
1486
1487
1488 for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
1489 afu->cmd[i].rcb.ctx_id = afu->ctx_hndl;
1490 afu->cmd[i].rcb.msi = SISL_MSI_RRQ_UPDATED;
1491 afu->cmd[i].rcb.rrq = 0x0;
1492 }
1493}
1494
1495
1496
1497
1498
1499static int init_global(struct cxlflash_cfg *cfg)
1500{
1501 struct afu *afu = cfg->afu;
1502 struct device *dev = &cfg->dev->dev;
1503 u64 wwpn[NUM_FC_PORTS];
1504 int i = 0, num_ports = 0;
1505 int rc = 0;
1506 u64 reg;
1507
1508 rc = read_vpd(cfg, &wwpn[0]);
1509 if (rc) {
1510 dev_err(dev, "%s: could not read vpd rc=%d\n", __func__, rc);
1511 goto out;
1512 }
1513
1514 pr_debug("%s: wwpn0=0x%llX wwpn1=0x%llX\n", __func__, wwpn[0], wwpn[1]);
1515
1516
1517 writeq_be((u64) afu->hrrq_start, &afu->host_map->rrq_start);
1518 writeq_be((u64) afu->hrrq_end, &afu->host_map->rrq_end);
1519
1520
1521 reg = readq_be(&afu->afu_map->global.regs.afu_config);
1522 reg |= SISL_AFUCONF_AR_ALL|SISL_AFUCONF_ENDIAN;
1523
1524
1525
1526
1527 writeq_be(reg, &afu->afu_map->global.regs.afu_config);
1528
1529
1530 if (afu->internal_lun) {
1531
1532 writeq_be(PORT0, &afu->afu_map->global.regs.afu_port_sel);
1533 num_ports = NUM_FC_PORTS - 1;
1534 } else {
1535 writeq_be(BOTH_PORTS, &afu->afu_map->global.regs.afu_port_sel);
1536 num_ports = NUM_FC_PORTS;
1537 }
1538
1539 for (i = 0; i < num_ports; i++) {
1540
1541 writeq_be(0, &afu->afu_map->global.fc_regs[i][FC_ERRMSK / 8]);
1542
1543 (void)readq_be(&afu->afu_map->global.
1544 fc_regs[i][FC_CNT_CRCERR / 8]);
1545 writeq_be(MC_CRC_THRESH, &afu->afu_map->global.fc_regs[i]
1546 [FC_CRC_THRESH / 8]);
1547
1548
1549 if (wwpn[i] != 0 &&
1550 afu_set_wwpn(afu, i,
1551 &afu->afu_map->global.fc_regs[i][0],
1552 wwpn[i])) {
1553 dev_err(dev, "%s: failed to set WWPN on port %d\n",
1554 __func__, i);
1555 rc = -EIO;
1556 goto out;
1557 }
1558
1559
1560
1561 msleep(100);
1562 }
1563
1564
1565
1566
1567 (void)readq_be(&afu->ctrl_map->mbox_r);
1568 writeq_be((SISL_CTX_CAP_REAL_MODE | SISL_CTX_CAP_HOST_XLATE |
1569 SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD |
1570 SISL_CTX_CAP_AFU_CMD | SISL_CTX_CAP_GSCSI_CMD),
1571 &afu->ctrl_map->ctx_cap);
1572
1573 afu->hb = readq_be(&afu->afu_map->global.regs.afu_hb);
1574
1575out:
1576 return rc;
1577}
1578
1579
1580
1581
1582
1583static int start_afu(struct cxlflash_cfg *cfg)
1584{
1585 struct afu *afu = cfg->afu;
1586 struct afu_cmd *cmd;
1587
1588 int i = 0;
1589 int rc = 0;
1590
1591 for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
1592 cmd = &afu->cmd[i];
1593
1594 init_completion(&cmd->cevent);
1595 spin_lock_init(&cmd->slock);
1596 cmd->parent = afu;
1597 }
1598
1599 init_pcr(cfg);
1600
1601
1602 memset(&afu->rrq_entry, 0, sizeof(afu->rrq_entry));
1603
1604
1605 afu->hrrq_start = &afu->rrq_entry[0];
1606 afu->hrrq_end = &afu->rrq_entry[NUM_RRQ_ENTRY - 1];
1607 afu->hrrq_curr = afu->hrrq_start;
1608 afu->toggle = 1;
1609
1610 rc = init_global(cfg);
1611
1612 pr_debug("%s: returning rc=%d\n", __func__, rc);
1613 return rc;
1614}
1615
1616
1617
1618
1619
1620
1621
1622static int init_mc(struct cxlflash_cfg *cfg)
1623{
1624 struct cxl_context *ctx;
1625 struct device *dev = &cfg->dev->dev;
1626 struct afu *afu = cfg->afu;
1627 int rc = 0;
1628 enum undo_level level;
1629
1630 ctx = cxl_get_context(cfg->dev);
1631 if (unlikely(!ctx))
1632 return -ENOMEM;
1633 cfg->mcctx = ctx;
1634
1635
1636 cxl_set_master(ctx);
1637
1638
1639 rc = cxl_afu_reset(cfg->mcctx);
1640 if (unlikely(rc)) {
1641 dev_err(dev, "%s: initial AFU reset failed rc=%d\n",
1642 __func__, rc);
1643 level = RELEASE_CONTEXT;
1644 goto out;
1645 }
1646
1647 rc = cxl_allocate_afu_irqs(ctx, 3);
1648 if (unlikely(rc)) {
1649 dev_err(dev, "%s: call to allocate_afu_irqs failed rc=%d!\n",
1650 __func__, rc);
1651 level = RELEASE_CONTEXT;
1652 goto out;
1653 }
1654
1655 rc = cxl_map_afu_irq(ctx, 1, cxlflash_sync_err_irq, afu,
1656 "SISL_MSI_SYNC_ERROR");
1657 if (unlikely(rc <= 0)) {
1658 dev_err(dev, "%s: IRQ 1 (SISL_MSI_SYNC_ERROR) map failed!\n",
1659 __func__);
1660 level = FREE_IRQ;
1661 goto out;
1662 }
1663
1664 rc = cxl_map_afu_irq(ctx, 2, cxlflash_rrq_irq, afu,
1665 "SISL_MSI_RRQ_UPDATED");
1666 if (unlikely(rc <= 0)) {
1667 dev_err(dev, "%s: IRQ 2 (SISL_MSI_RRQ_UPDATED) map failed!\n",
1668 __func__);
1669 level = UNMAP_ONE;
1670 goto out;
1671 }
1672
1673 rc = cxl_map_afu_irq(ctx, 3, cxlflash_async_err_irq, afu,
1674 "SISL_MSI_ASYNC_ERROR");
1675 if (unlikely(rc <= 0)) {
1676 dev_err(dev, "%s: IRQ 3 (SISL_MSI_ASYNC_ERROR) map failed!\n",
1677 __func__);
1678 level = UNMAP_TWO;
1679 goto out;
1680 }
1681
1682 rc = 0;
1683
1684
1685
1686
1687
1688 rc = start_context(cfg);
1689 if (unlikely(rc)) {
1690 dev_err(dev, "%s: start context failed rc=%d\n", __func__, rc);
1691 level = UNMAP_THREE;
1692 goto out;
1693 }
1694ret:
1695 pr_debug("%s: returning rc=%d\n", __func__, rc);
1696 return rc;
1697out:
1698 term_mc(cfg, level);
1699 goto ret;
1700}
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711static int init_afu(struct cxlflash_cfg *cfg)
1712{
1713 u64 reg;
1714 int rc = 0;
1715 struct afu *afu = cfg->afu;
1716 struct device *dev = &cfg->dev->dev;
1717
1718 cxl_perst_reloads_same_image(cfg->cxl_afu, true);
1719
1720 rc = init_mc(cfg);
1721 if (rc) {
1722 dev_err(dev, "%s: call to init_mc failed, rc=%d!\n",
1723 __func__, rc);
1724 goto out;
1725 }
1726
1727
1728 afu->afu_map = cxl_psa_map(cfg->mcctx);
1729 if (!afu->afu_map) {
1730 dev_err(dev, "%s: call to cxl_psa_map failed!\n", __func__);
1731 rc = -ENOMEM;
1732 goto err1;
1733 }
1734
1735
1736 reg = readq(&afu->afu_map->global.regs.afu_version);
1737 memcpy(afu->version, ®, sizeof(reg));
1738 afu->interface_version =
1739 readq_be(&afu->afu_map->global.regs.interface_version);
1740 if ((afu->interface_version + 1) == 0) {
1741 pr_err("Back level AFU, please upgrade. AFU version %s "
1742 "interface version 0x%llx\n", afu->version,
1743 afu->interface_version);
1744 rc = -EINVAL;
1745 goto err2;
1746 }
1747
1748 pr_debug("%s: afu version %s, interface version 0x%llX\n", __func__,
1749 afu->version, afu->interface_version);
1750
1751 rc = start_afu(cfg);
1752 if (rc) {
1753 dev_err(dev, "%s: call to start_afu failed, rc=%d!\n",
1754 __func__, rc);
1755 goto err2;
1756 }
1757
1758 afu_err_intr_init(cfg->afu);
1759 atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room));
1760
1761
1762 cxlflash_restore_luntable(cfg);
1763out:
1764 pr_debug("%s: returning rc=%d\n", __func__, rc);
1765 return rc;
1766
1767err2:
1768 cxl_psa_unmap((void __iomem *)afu->afu_map);
1769 afu->afu_map = NULL;
1770err1:
1771 term_mc(cfg, UNDO_START);
1772 goto out;
1773}
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx_hndl_u,
1798 res_hndl_t res_hndl_u, u8 mode)
1799{
1800 struct cxlflash_cfg *cfg = afu->parent;
1801 struct device *dev = &cfg->dev->dev;
1802 struct afu_cmd *cmd = NULL;
1803 int rc = 0;
1804 int retry_cnt = 0;
1805 static DEFINE_MUTEX(sync_active);
1806
1807 if (cfg->state != STATE_NORMAL) {
1808 pr_debug("%s: Sync not required! (%u)\n", __func__, cfg->state);
1809 return 0;
1810 }
1811
1812 mutex_lock(&sync_active);
1813retry:
1814 cmd = cmd_checkout(afu);
1815 if (unlikely(!cmd)) {
1816 retry_cnt++;
1817 udelay(1000 * retry_cnt);
1818 if (retry_cnt < MC_RETRY_CNT)
1819 goto retry;
1820 dev_err(dev, "%s: could not get a free command\n", __func__);
1821 rc = -1;
1822 goto out;
1823 }
1824
1825 pr_debug("%s: afu=%p cmd=%p %d\n", __func__, afu, cmd, ctx_hndl_u);
1826
1827 memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb));
1828
1829 cmd->rcb.req_flags = SISL_REQ_FLAGS_AFU_CMD;
1830 cmd->rcb.port_sel = 0x0;
1831 cmd->rcb.lun_id = 0x0;
1832 cmd->rcb.data_len = 0x0;
1833 cmd->rcb.data_ea = 0x0;
1834 cmd->rcb.timeout = MC_AFU_SYNC_TIMEOUT;
1835
1836 cmd->rcb.cdb[0] = 0xC0;
1837 cmd->rcb.cdb[1] = mode;
1838
1839
1840 *((__be16 *)&cmd->rcb.cdb[2]) = cpu_to_be16(ctx_hndl_u);
1841 *((__be32 *)&cmd->rcb.cdb[4]) = cpu_to_be32(res_hndl_u);
1842
1843 rc = send_cmd(afu, cmd);
1844 if (unlikely(rc))
1845 goto out;
1846
1847 wait_resp(afu, cmd);
1848
1849
1850 if (unlikely((cmd->sa.ioasc != 0) ||
1851 (cmd->sa.host_use_b[0] & B_ERROR)))
1852 rc = -1;
1853out:
1854 mutex_unlock(&sync_active);
1855 if (cmd)
1856 cmd_checkin(cmd);
1857 pr_debug("%s: returning rc=%d\n", __func__, rc);
1858 return rc;
1859}
1860
1861
1862
1863
1864
1865
1866
1867static int afu_reset(struct cxlflash_cfg *cfg)
1868{
1869 int rc = 0;
1870
1871
1872
1873
1874 term_afu(cfg);
1875
1876 rc = init_afu(cfg);
1877
1878 pr_debug("%s: returning rc=%d\n", __func__, rc);
1879 return rc;
1880}
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890static int cxlflash_eh_device_reset_handler(struct scsi_cmnd *scp)
1891{
1892 int rc = SUCCESS;
1893 struct Scsi_Host *host = scp->device->host;
1894 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
1895 struct afu *afu = cfg->afu;
1896 int rcr = 0;
1897
1898 pr_debug("%s: (scp=%p) %d/%d/%d/%llu "
1899 "cdb=(%08X-%08X-%08X-%08X)\n", __func__, scp,
1900 host->host_no, scp->device->channel,
1901 scp->device->id, scp->device->lun,
1902 get_unaligned_be32(&((u32 *)scp->cmnd)[0]),
1903 get_unaligned_be32(&((u32 *)scp->cmnd)[1]),
1904 get_unaligned_be32(&((u32 *)scp->cmnd)[2]),
1905 get_unaligned_be32(&((u32 *)scp->cmnd)[3]));
1906
1907retry:
1908 switch (cfg->state) {
1909 case STATE_NORMAL:
1910 rcr = send_tmf(afu, scp, TMF_LUN_RESET);
1911 if (unlikely(rcr))
1912 rc = FAILED;
1913 break;
1914 case STATE_RESET:
1915 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
1916 goto retry;
1917 default:
1918 rc = FAILED;
1919 break;
1920 }
1921
1922 pr_debug("%s: returning rc=%d\n", __func__, rc);
1923 return rc;
1924}
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934static int cxlflash_eh_host_reset_handler(struct scsi_cmnd *scp)
1935{
1936 int rc = SUCCESS;
1937 int rcr = 0;
1938 struct Scsi_Host *host = scp->device->host;
1939 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
1940
1941 pr_debug("%s: (scp=%p) %d/%d/%d/%llu "
1942 "cdb=(%08X-%08X-%08X-%08X)\n", __func__, scp,
1943 host->host_no, scp->device->channel,
1944 scp->device->id, scp->device->lun,
1945 get_unaligned_be32(&((u32 *)scp->cmnd)[0]),
1946 get_unaligned_be32(&((u32 *)scp->cmnd)[1]),
1947 get_unaligned_be32(&((u32 *)scp->cmnd)[2]),
1948 get_unaligned_be32(&((u32 *)scp->cmnd)[3]));
1949
1950 switch (cfg->state) {
1951 case STATE_NORMAL:
1952 cfg->state = STATE_RESET;
1953 cxlflash_mark_contexts_error(cfg);
1954 rcr = afu_reset(cfg);
1955 if (rcr) {
1956 rc = FAILED;
1957 cfg->state = STATE_FAILTERM;
1958 } else
1959 cfg->state = STATE_NORMAL;
1960 wake_up_all(&cfg->reset_waitq);
1961 break;
1962 case STATE_RESET:
1963 wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
1964 if (cfg->state == STATE_NORMAL)
1965 break;
1966
1967 default:
1968 rc = FAILED;
1969 break;
1970 }
1971
1972 pr_debug("%s: returning rc=%d\n", __func__, rc);
1973 return rc;
1974}
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985static int cxlflash_change_queue_depth(struct scsi_device *sdev, int qdepth)
1986{
1987
1988 if (qdepth > CXLFLASH_MAX_CMDS_PER_LUN)
1989 qdepth = CXLFLASH_MAX_CMDS_PER_LUN;
1990
1991 scsi_change_queue_depth(sdev, qdepth);
1992 return sdev->queue_depth;
1993}
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003static ssize_t cxlflash_show_port_status(u32 port, struct afu *afu, char *buf)
2004{
2005 char *disp_status;
2006 u64 status;
2007 __be64 __iomem *fc_regs;
2008
2009 if (port >= NUM_FC_PORTS)
2010 return 0;
2011
2012 fc_regs = &afu->afu_map->global.fc_regs[port][0];
2013 status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]);
2014 status &= FC_MTIP_STATUS_MASK;
2015
2016 if (status == FC_MTIP_STATUS_ONLINE)
2017 disp_status = "online";
2018 else if (status == FC_MTIP_STATUS_OFFLINE)
2019 disp_status = "offline";
2020 else
2021 disp_status = "unknown";
2022
2023 return scnprintf(buf, PAGE_SIZE, "%s\n", disp_status);
2024}
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034static ssize_t port0_show(struct device *dev,
2035 struct device_attribute *attr,
2036 char *buf)
2037{
2038 struct Scsi_Host *shost = class_to_shost(dev);
2039 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2040 struct afu *afu = cfg->afu;
2041
2042 return cxlflash_show_port_status(0, afu, buf);
2043}
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053static ssize_t port1_show(struct device *dev,
2054 struct device_attribute *attr,
2055 char *buf)
2056{
2057 struct Scsi_Host *shost = class_to_shost(dev);
2058 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2059 struct afu *afu = cfg->afu;
2060
2061 return cxlflash_show_port_status(1, afu, buf);
2062}
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072static ssize_t lun_mode_show(struct device *dev,
2073 struct device_attribute *attr, char *buf)
2074{
2075 struct Scsi_Host *shost = class_to_shost(dev);
2076 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2077 struct afu *afu = cfg->afu;
2078
2079 return scnprintf(buf, PAGE_SIZE, "%u\n", afu->internal_lun);
2080}
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104static ssize_t lun_mode_store(struct device *dev,
2105 struct device_attribute *attr,
2106 const char *buf, size_t count)
2107{
2108 struct Scsi_Host *shost = class_to_shost(dev);
2109 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2110 struct afu *afu = cfg->afu;
2111 int rc;
2112 u32 lun_mode;
2113
2114 rc = kstrtouint(buf, 10, &lun_mode);
2115 if (!rc && (lun_mode < 5) && (lun_mode != afu->internal_lun)) {
2116 afu->internal_lun = lun_mode;
2117 afu_reset(cfg);
2118 scsi_scan_host(cfg->host);
2119 }
2120
2121 return count;
2122}
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132static ssize_t ioctl_version_show(struct device *dev,
2133 struct device_attribute *attr, char *buf)
2134{
2135 return scnprintf(buf, PAGE_SIZE, "%u\n", DK_CXLFLASH_VERSION_0);
2136}
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146static ssize_t cxlflash_show_port_lun_table(u32 port,
2147 struct afu *afu,
2148 char *buf)
2149{
2150 int i;
2151 ssize_t bytes = 0;
2152 __be64 __iomem *fc_port;
2153
2154 if (port >= NUM_FC_PORTS)
2155 return 0;
2156
2157 fc_port = &afu->afu_map->global.fc_port[port][0];
2158
2159 for (i = 0; i < CXLFLASH_NUM_VLUNS; i++)
2160 bytes += scnprintf(buf + bytes, PAGE_SIZE - bytes,
2161 "%03d: %016llX\n", i, readq_be(&fc_port[i]));
2162 return bytes;
2163}
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173static ssize_t port0_lun_table_show(struct device *dev,
2174 struct device_attribute *attr,
2175 char *buf)
2176{
2177 struct Scsi_Host *shost = class_to_shost(dev);
2178 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2179 struct afu *afu = cfg->afu;
2180
2181 return cxlflash_show_port_lun_table(0, afu, buf);
2182}
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192static ssize_t port1_lun_table_show(struct device *dev,
2193 struct device_attribute *attr,
2194 char *buf)
2195{
2196 struct Scsi_Host *shost = class_to_shost(dev);
2197 struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)shost->hostdata;
2198 struct afu *afu = cfg->afu;
2199
2200 return cxlflash_show_port_lun_table(1, afu, buf);
2201}
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211static ssize_t mode_show(struct device *dev,
2212 struct device_attribute *attr, char *buf)
2213{
2214 struct scsi_device *sdev = to_scsi_device(dev);
2215
2216 return scnprintf(buf, PAGE_SIZE, "%s\n",
2217 sdev->hostdata ? "superpipe" : "legacy");
2218}
2219
2220
2221
2222
2223static DEVICE_ATTR_RO(port0);
2224static DEVICE_ATTR_RO(port1);
2225static DEVICE_ATTR_RW(lun_mode);
2226static DEVICE_ATTR_RO(ioctl_version);
2227static DEVICE_ATTR_RO(port0_lun_table);
2228static DEVICE_ATTR_RO(port1_lun_table);
2229
2230static struct device_attribute *cxlflash_host_attrs[] = {
2231 &dev_attr_port0,
2232 &dev_attr_port1,
2233 &dev_attr_lun_mode,
2234 &dev_attr_ioctl_version,
2235 &dev_attr_port0_lun_table,
2236 &dev_attr_port1_lun_table,
2237 NULL
2238};
2239
2240
2241
2242
2243static DEVICE_ATTR_RO(mode);
2244
2245static struct device_attribute *cxlflash_dev_attrs[] = {
2246 &dev_attr_mode,
2247 NULL
2248};
2249
2250
2251
2252
2253static struct scsi_host_template driver_template = {
2254 .module = THIS_MODULE,
2255 .name = CXLFLASH_ADAPTER_NAME,
2256 .info = cxlflash_driver_info,
2257 .ioctl = cxlflash_ioctl,
2258 .proc_name = CXLFLASH_NAME,
2259 .queuecommand = cxlflash_queuecommand,
2260 .eh_device_reset_handler = cxlflash_eh_device_reset_handler,
2261 .eh_host_reset_handler = cxlflash_eh_host_reset_handler,
2262 .change_queue_depth = cxlflash_change_queue_depth,
2263 .cmd_per_lun = 16,
2264 .can_queue = CXLFLASH_MAX_CMDS,
2265 .this_id = -1,
2266 .sg_tablesize = SG_NONE,
2267 .max_sectors = CXLFLASH_MAX_SECTORS,
2268 .use_clustering = ENABLE_CLUSTERING,
2269 .shost_attrs = cxlflash_host_attrs,
2270 .sdev_attrs = cxlflash_dev_attrs,
2271};
2272
2273
2274
2275
2276static struct dev_dependent_vals dev_corsa_vals = { CXLFLASH_MAX_SECTORS };
2277
2278
2279
2280
2281static struct pci_device_id cxlflash_pci_table[] = {
2282 {PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CORSA,
2283 PCI_ANY_ID, PCI_ANY_ID, 0, 0, (kernel_ulong_t)&dev_corsa_vals},
2284 {}
2285};
2286
2287MODULE_DEVICE_TABLE(pci, cxlflash_pci_table);
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299static void cxlflash_worker_thread(struct work_struct *work)
2300{
2301 struct cxlflash_cfg *cfg = container_of(work, struct cxlflash_cfg,
2302 work_q);
2303 struct afu *afu = cfg->afu;
2304 struct device *dev = &cfg->dev->dev;
2305 int port;
2306 ulong lock_flags;
2307
2308
2309
2310 if (cfg->state != STATE_NORMAL)
2311 return;
2312
2313 spin_lock_irqsave(cfg->host->host_lock, lock_flags);
2314
2315 if (cfg->lr_state == LINK_RESET_REQUIRED) {
2316 port = cfg->lr_port;
2317 if (port < 0)
2318 dev_err(dev, "%s: invalid port index %d\n",
2319 __func__, port);
2320 else {
2321 spin_unlock_irqrestore(cfg->host->host_lock,
2322 lock_flags);
2323
2324
2325 afu_link_reset(afu, port,
2326 &afu->afu_map->global.fc_regs[port][0]);
2327 spin_lock_irqsave(cfg->host->host_lock, lock_flags);
2328 }
2329
2330 cfg->lr_state = LINK_RESET_COMPLETE;
2331 }
2332
2333 if (afu->read_room) {
2334 atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room));
2335 afu->read_room = false;
2336 }
2337
2338 spin_unlock_irqrestore(cfg->host->host_lock, lock_flags);
2339
2340 if (atomic_dec_if_positive(&cfg->scan_host_needed) >= 0)
2341 scsi_scan_host(cfg->host);
2342}
2343
2344
2345
2346
2347
2348
2349
2350
2351static int cxlflash_probe(struct pci_dev *pdev,
2352 const struct pci_device_id *dev_id)
2353{
2354 struct Scsi_Host *host;
2355 struct cxlflash_cfg *cfg = NULL;
2356 struct device *phys_dev;
2357 struct dev_dependent_vals *ddv;
2358 int rc = 0;
2359
2360 dev_dbg(&pdev->dev, "%s: Found CXLFLASH with IRQ: %d\n",
2361 __func__, pdev->irq);
2362
2363 ddv = (struct dev_dependent_vals *)dev_id->driver_data;
2364 driver_template.max_sectors = ddv->max_sectors;
2365
2366 host = scsi_host_alloc(&driver_template, sizeof(struct cxlflash_cfg));
2367 if (!host) {
2368 dev_err(&pdev->dev, "%s: call to scsi_host_alloc failed!\n",
2369 __func__);
2370 rc = -ENOMEM;
2371 goto out;
2372 }
2373
2374 host->max_id = CXLFLASH_MAX_NUM_TARGETS_PER_BUS;
2375 host->max_lun = CXLFLASH_MAX_NUM_LUNS_PER_TARGET;
2376 host->max_channel = NUM_FC_PORTS - 1;
2377 host->unique_id = host->host_no;
2378 host->max_cmd_len = CXLFLASH_MAX_CDB_LEN;
2379
2380 cfg = (struct cxlflash_cfg *)host->hostdata;
2381 cfg->host = host;
2382 rc = alloc_mem(cfg);
2383 if (rc) {
2384 dev_err(&pdev->dev, "%s: call to alloc_mem failed!\n",
2385 __func__);
2386 rc = -ENOMEM;
2387 scsi_host_put(cfg->host);
2388 goto out;
2389 }
2390
2391 cfg->init_state = INIT_STATE_NONE;
2392 cfg->dev = pdev;
2393 cfg->cxl_fops = cxlflash_cxl_fops;
2394
2395
2396
2397
2398
2399
2400
2401 cfg->promote_lun_index = 0;
2402 cfg->last_lun_index[0] = CXLFLASH_NUM_VLUNS/2 - 1;
2403 cfg->last_lun_index[1] = CXLFLASH_NUM_VLUNS/2 - 1;
2404
2405 cfg->dev_id = (struct pci_device_id *)dev_id;
2406
2407 init_waitqueue_head(&cfg->tmf_waitq);
2408 init_waitqueue_head(&cfg->reset_waitq);
2409
2410 INIT_WORK(&cfg->work_q, cxlflash_worker_thread);
2411 cfg->lr_state = LINK_RESET_INVALID;
2412 cfg->lr_port = -1;
2413 spin_lock_init(&cfg->tmf_slock);
2414 mutex_init(&cfg->ctx_tbl_list_mutex);
2415 mutex_init(&cfg->ctx_recovery_mutex);
2416 init_rwsem(&cfg->ioctl_rwsem);
2417 INIT_LIST_HEAD(&cfg->ctx_err_recovery);
2418 INIT_LIST_HEAD(&cfg->lluns);
2419
2420 pci_set_drvdata(pdev, cfg);
2421
2422
2423
2424
2425
2426
2427 phys_dev = cxl_get_phys_dev(pdev);
2428 if (!dev_is_pci(phys_dev)) {
2429 dev_err(&pdev->dev, "%s: not a pci dev\n", __func__);
2430 rc = -ENODEV;
2431 goto out_remove;
2432 }
2433 cfg->parent_dev = to_pci_dev(phys_dev);
2434
2435 cfg->cxl_afu = cxl_pci_to_afu(pdev);
2436
2437 rc = init_pci(cfg);
2438 if (rc) {
2439 dev_err(&pdev->dev, "%s: call to init_pci "
2440 "failed rc=%d!\n", __func__, rc);
2441 goto out_remove;
2442 }
2443 cfg->init_state = INIT_STATE_PCI;
2444
2445 rc = init_afu(cfg);
2446 if (rc) {
2447 dev_err(&pdev->dev, "%s: call to init_afu "
2448 "failed rc=%d!\n", __func__, rc);
2449 goto out_remove;
2450 }
2451 cfg->init_state = INIT_STATE_AFU;
2452
2453 rc = init_scsi(cfg);
2454 if (rc) {
2455 dev_err(&pdev->dev, "%s: call to init_scsi "
2456 "failed rc=%d!\n", __func__, rc);
2457 goto out_remove;
2458 }
2459 cfg->init_state = INIT_STATE_SCSI;
2460
2461out:
2462 pr_debug("%s: returning rc=%d\n", __func__, rc);
2463 return rc;
2464
2465out_remove:
2466 cxlflash_remove(pdev);
2467 goto out;
2468}
2469
2470
2471
2472
2473
2474
2475
2476
2477static void drain_ioctls(struct cxlflash_cfg *cfg)
2478{
2479 down_write(&cfg->ioctl_rwsem);
2480 up_write(&cfg->ioctl_rwsem);
2481}
2482
2483
2484
2485
2486
2487
2488
2489
2490static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev,
2491 pci_channel_state_t state)
2492{
2493 int rc = 0;
2494 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
2495 struct device *dev = &cfg->dev->dev;
2496
2497 dev_dbg(dev, "%s: pdev=%p state=%u\n", __func__, pdev, state);
2498
2499 switch (state) {
2500 case pci_channel_io_frozen:
2501 cfg->state = STATE_RESET;
2502 scsi_block_requests(cfg->host);
2503 drain_ioctls(cfg);
2504 rc = cxlflash_mark_contexts_error(cfg);
2505 if (unlikely(rc))
2506 dev_err(dev, "%s: Failed to mark user contexts!(%d)\n",
2507 __func__, rc);
2508 term_mc(cfg, UNDO_START);
2509 stop_afu(cfg);
2510 return PCI_ERS_RESULT_NEED_RESET;
2511 case pci_channel_io_perm_failure:
2512 cfg->state = STATE_FAILTERM;
2513 wake_up_all(&cfg->reset_waitq);
2514 scsi_unblock_requests(cfg->host);
2515 return PCI_ERS_RESULT_DISCONNECT;
2516 default:
2517 break;
2518 }
2519 return PCI_ERS_RESULT_NEED_RESET;
2520}
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531static pci_ers_result_t cxlflash_pci_slot_reset(struct pci_dev *pdev)
2532{
2533 int rc = 0;
2534 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
2535 struct device *dev = &cfg->dev->dev;
2536
2537 dev_dbg(dev, "%s: pdev=%p\n", __func__, pdev);
2538
2539 rc = init_afu(cfg);
2540 if (unlikely(rc)) {
2541 dev_err(dev, "%s: EEH recovery failed! (%d)\n", __func__, rc);
2542 return PCI_ERS_RESULT_DISCONNECT;
2543 }
2544
2545 return PCI_ERS_RESULT_RECOVERED;
2546}
2547
2548
2549
2550
2551
2552static void cxlflash_pci_resume(struct pci_dev *pdev)
2553{
2554 struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
2555 struct device *dev = &cfg->dev->dev;
2556
2557 dev_dbg(dev, "%s: pdev=%p\n", __func__, pdev);
2558
2559 cfg->state = STATE_NORMAL;
2560 wake_up_all(&cfg->reset_waitq);
2561 scsi_unblock_requests(cfg->host);
2562}
2563
2564static const struct pci_error_handlers cxlflash_err_handler = {
2565 .error_detected = cxlflash_pci_error_detected,
2566 .slot_reset = cxlflash_pci_slot_reset,
2567 .resume = cxlflash_pci_resume,
2568};
2569
2570
2571
2572
2573static struct pci_driver cxlflash_driver = {
2574 .name = CXLFLASH_NAME,
2575 .id_table = cxlflash_pci_table,
2576 .probe = cxlflash_probe,
2577 .remove = cxlflash_remove,
2578 .err_handler = &cxlflash_err_handler,
2579};
2580
2581
2582
2583
2584
2585
2586static int __init init_cxlflash(void)
2587{
2588 pr_info("%s: IBM Power CXL Flash Adapter: %s\n",
2589 __func__, CXLFLASH_DRIVER_DATE);
2590
2591 cxlflash_list_init();
2592
2593 return pci_register_driver(&cxlflash_driver);
2594}
2595
2596
2597
2598
2599static void __exit exit_cxlflash(void)
2600{
2601 cxlflash_term_global_luns();
2602 cxlflash_free_errpage();
2603
2604 pci_unregister_driver(&cxlflash_driver);
2605}
2606
2607module_init(init_cxlflash);
2608module_exit(exit_cxlflash);
2609