1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/pci.h>
24#include <linux/slab.h>
25#include <linux/interrupt.h>
26#include <linux/delay.h>
27#include <asm/unaligned.h>
28#include <linux/crc-t10dif.h>
29#include <net/checksum.h>
30
31#include <scsi/scsi.h>
32#include <scsi/scsi_device.h>
33#include <scsi/scsi_eh.h>
34#include <scsi/scsi_host.h>
35#include <scsi/scsi_tcq.h>
36#include <scsi/scsi_transport_fc.h>
37#include <scsi/fc/fc_fs.h>
38
39#include "lpfc_version.h"
40#include "lpfc_hw4.h"
41#include "lpfc_hw.h"
42#include "lpfc_sli.h"
43#include "lpfc_sli4.h"
44#include "lpfc_nl.h"
45#include "lpfc_disc.h"
46#include "lpfc.h"
47#include "lpfc_nvme.h"
48#include "lpfc_scsi.h"
49#include "lpfc_logmsg.h"
50#include "lpfc_crtn.h"
51#include "lpfc_vport.h"
52#include "lpfc_debugfs.h"
53
54
55
56static struct lpfc_io_buf *
57lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 int idx, int expedite);
59
60static void
61lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62
63static struct nvme_fc_port_template lpfc_nvme_template;
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81static int
82lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
83 unsigned int qidx, u16 qsize,
84 void **handle)
85{
86 struct lpfc_nvme_lport *lport;
87 struct lpfc_vport *vport;
88 struct lpfc_nvme_qhandle *qhandle;
89 char *str;
90
91 if (!pnvme_lport->private)
92 return -ENOMEM;
93
94 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
95 vport = lport->vport;
96 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
97 if (qhandle == NULL)
98 return -ENOMEM;
99
100 qhandle->cpu_id = raw_smp_processor_id();
101 qhandle->qidx = qidx;
102
103
104
105
106
107 if (qidx) {
108 str = "IO ";
109 qhandle->index = ((qidx - 1) %
110 lpfc_nvme_template.max_hw_queues);
111 } else {
112 str = "ADM";
113 qhandle->index = qidx;
114 }
115
116 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
117 "6073 Binding %s HdwQueue %d (cpu %d) to "
118 "hdw_queue %d qhandle x%px\n", str,
119 qidx, qhandle->cpu_id, qhandle->index, qhandle);
120 *handle = (void *)qhandle;
121 return 0;
122}
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138static void
139lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
140 unsigned int qidx,
141 void *handle)
142{
143 struct lpfc_nvme_lport *lport;
144 struct lpfc_vport *vport;
145
146 if (!pnvme_lport->private)
147 return;
148
149 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
150 vport = lport->vport;
151
152 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
153 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
154 lport, qidx, handle);
155 kfree(handle);
156}
157
158static void
159lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
160{
161 struct lpfc_nvme_lport *lport = localport->private;
162
163 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
164 "6173 localport x%px delete complete\n",
165 lport);
166
167
168 if (lport->vport->localport)
169 complete(lport->lport_unreg_cmp);
170}
171
172
173
174
175
176
177
178
179
180
181
182
183static void
184lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
185{
186 struct lpfc_nvme_rport *rport = remoteport->private;
187 struct lpfc_vport *vport;
188 struct lpfc_nodelist *ndlp;
189 u32 fc4_xpt_flags;
190
191 ndlp = rport->ndlp;
192 if (!ndlp) {
193 pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
194 __func__, rport, remoteport);
195 goto rport_err;
196 }
197
198 vport = ndlp->vport;
199 if (!vport) {
200 pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
201 __func__, ndlp, ndlp->nlp_state, rport);
202 goto rport_err;
203 }
204
205 fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
206
207
208
209
210
211 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
212 "6146 remoteport delete of remoteport x%px\n",
213 remoteport);
214 spin_lock_irq(&ndlp->lock);
215
216
217
218
219 if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
220 ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
221
222 spin_unlock_irq(&ndlp->lock);
223
224
225
226
227
228 if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
229 lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
230
231 rport_err:
232 return;
233}
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251int
252lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
253 struct lpfc_async_xchg_ctx *axchg)
254{
255#if (IS_ENABLED(CONFIG_NVME_FC))
256 struct lpfc_vport *vport;
257 struct lpfc_nvme_rport *lpfc_rport;
258 struct nvme_fc_remote_port *remoteport;
259 struct lpfc_nvme_lport *lport;
260 uint32_t *payload = axchg->payload;
261 int rc;
262
263 vport = axchg->ndlp->vport;
264 lpfc_rport = axchg->ndlp->nrport;
265 if (!lpfc_rport)
266 return -EINVAL;
267
268 remoteport = lpfc_rport->remoteport;
269 if (!vport->localport)
270 return -EINVAL;
271
272 lport = vport->localport->private;
273 if (!lport)
274 return -EINVAL;
275
276 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
277 axchg->size);
278
279 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
280 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
281 "%08x %08x %08x\n",
282 axchg->size, rc,
283 *payload, *(payload+1), *(payload+2),
284 *(payload+3), *(payload+4), *(payload+5));
285
286 if (!rc)
287 return 0;
288#endif
289 return 1;
290}
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305void
306__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
307 struct lpfc_iocbq *cmdwqe,
308 struct lpfc_wcqe_complete *wcqe)
309{
310 struct nvmefc_ls_req *pnvme_lsreq;
311 struct lpfc_dmabuf *buf_ptr;
312 struct lpfc_nodelist *ndlp;
313 uint32_t status;
314
315 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
316 ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
317 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
318
319 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
320 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
321 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
322 "ndlp:x%px\n",
323 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
324 cmdwqe->sli4_xritag, status,
325 (wcqe->parameter & 0xffff),
326 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
327
328 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
329 cmdwqe->sli4_xritag, status, wcqe->parameter);
330
331 if (cmdwqe->context3) {
332 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
333 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
334 kfree(buf_ptr);
335 cmdwqe->context3 = NULL;
336 }
337 if (pnvme_lsreq->done)
338 pnvme_lsreq->done(pnvme_lsreq, status);
339 else
340 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
341 "6046 NVMEx cmpl without done call back? "
342 "Data x%px DID %x Xri: %x status %x\n",
343 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
344 cmdwqe->sli4_xritag, status);
345 if (ndlp) {
346 lpfc_nlp_put(ndlp);
347 cmdwqe->context1 = NULL;
348 }
349 lpfc_sli_release_iocbq(phba, cmdwqe);
350}
351
352static void
353lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
354 struct lpfc_wcqe_complete *wcqe)
355{
356 struct lpfc_vport *vport = cmdwqe->vport;
357 struct lpfc_nvme_lport *lport;
358 uint32_t status;
359
360 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
361
362 if (vport->localport) {
363 lport = (struct lpfc_nvme_lport *)vport->localport->private;
364 if (lport) {
365 atomic_inc(&lport->fc4NvmeLsCmpls);
366 if (status) {
367 if (bf_get(lpfc_wcqe_c_xb, wcqe))
368 atomic_inc(&lport->cmpl_ls_xb);
369 atomic_inc(&lport->cmpl_ls_err);
370 }
371 }
372 }
373
374 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
375}
376
377static int
378lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
379 struct lpfc_dmabuf *inp,
380 struct nvmefc_ls_req *pnvme_lsreq,
381 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
382 struct lpfc_wcqe_complete *),
383 struct lpfc_nodelist *ndlp, uint32_t num_entry,
384 uint32_t tmo, uint8_t retry)
385{
386 struct lpfc_hba *phba = vport->phba;
387 union lpfc_wqe128 *wqe;
388 struct lpfc_iocbq *genwqe;
389 struct ulp_bde64 *bpl;
390 struct ulp_bde64 bde;
391 int i, rc, xmit_len, first_len;
392
393
394 genwqe = lpfc_sli_get_iocbq(phba);
395 if (genwqe == NULL)
396 return 1;
397
398 wqe = &genwqe->wqe;
399
400 memset(wqe, 0, sizeof(union lpfc_wqe));
401
402 genwqe->context3 = (uint8_t *)bmp;
403 genwqe->iocb_flag |= LPFC_IO_NVME_LS;
404
405
406 genwqe->context1 = lpfc_nlp_get(ndlp);
407 if (!genwqe->context1) {
408 dev_warn(&phba->pcidev->dev,
409 "Warning: Failed node ref, not sending LS_REQ\n");
410 lpfc_sli_release_iocbq(phba, genwqe);
411 return 1;
412 }
413
414 genwqe->context2 = (uint8_t *)pnvme_lsreq;
415
416
417 if (!tmo)
418
419 tmo = (3 * phba->fc_ratov);
420
421
422 xmit_len = 0;
423 first_len = 0;
424 bpl = (struct ulp_bde64 *)bmp->virt;
425 for (i = 0; i < num_entry; i++) {
426 bde.tus.w = bpl[i].tus.w;
427 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
428 break;
429 xmit_len += bde.tus.f.bdeSize;
430 if (i == 0)
431 first_len = xmit_len;
432 }
433
434 genwqe->rsvd2 = num_entry;
435 genwqe->hba_wqidx = 0;
436
437
438 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
439 wqe->generic.bde.tus.f.bdeSize = first_len;
440 wqe->generic.bde.addrLow = bpl[0].addrLow;
441 wqe->generic.bde.addrHigh = bpl[0].addrHigh;
442
443
444 wqe->gen_req.request_payload_len = first_len;
445
446
447
448
449 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
450 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
451 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
452 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
453 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
454
455
456 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
457 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
458 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
459
460
461 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
462 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
463 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
464 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
465
466
467 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
468
469
470 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
471
472
473 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
474 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
475 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
476 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
477 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
478
479
480 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
481 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
482
483
484
485 genwqe->wqe_cmpl = cmpl;
486 genwqe->iocb_cmpl = NULL;
487 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
488 genwqe->vport = vport;
489 genwqe->retry = retry;
490
491 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
492 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
493
494 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
495 if (rc) {
496 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
497 "6045 Issue GEN REQ WQE to NPORT x%x "
498 "Data: x%x x%x rc x%x\n",
499 ndlp->nlp_DID, genwqe->iotag,
500 vport->port_state, rc);
501 lpfc_nlp_put(ndlp);
502 lpfc_sli_release_iocbq(phba, genwqe);
503 return 1;
504 }
505
506 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
507 "6050 Issue GEN REQ WQE to NPORT x%x "
508 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
509 "bmp:x%px xmit:%d 1st:%d\n",
510 ndlp->nlp_DID, genwqe->sli4_xritag,
511 vport->port_state,
512 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
513 return 0;
514}
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531int
532__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
533 struct nvmefc_ls_req *pnvme_lsreq,
534 void (*gen_req_cmp)(struct lpfc_hba *phba,
535 struct lpfc_iocbq *cmdwqe,
536 struct lpfc_wcqe_complete *wcqe))
537{
538 struct lpfc_dmabuf *bmp;
539 struct ulp_bde64 *bpl;
540 int ret;
541 uint16_t ntype, nstate;
542
543 if (!ndlp) {
544 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
545 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
546 "LS Req\n",
547 ndlp);
548 return -ENODEV;
549 }
550
551 ntype = ndlp->nlp_type;
552 nstate = ndlp->nlp_state;
553 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
554 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
555 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
556 "6088 NVMEx LS REQ: Fail DID x%06x not "
557 "ready for IO. Type x%x, State x%x\n",
558 ndlp->nlp_DID, ntype, nstate);
559 return -ENODEV;
560 }
561
562 if (!vport->phba->sli4_hba.nvmels_wq)
563 return -ENOMEM;
564
565
566
567
568
569
570
571
572
573
574
575
576
577 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
578 if (!bmp) {
579 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
580 "6044 NVMEx LS REQ: Could not alloc LS buf "
581 "for DID %x\n",
582 ndlp->nlp_DID);
583 return -ENOMEM;
584 }
585
586 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
587 if (!bmp->virt) {
588 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
589 "6042 NVMEx LS REQ: Could not alloc mbuf "
590 "for DID %x\n",
591 ndlp->nlp_DID);
592 kfree(bmp);
593 return -ENOMEM;
594 }
595
596 INIT_LIST_HEAD(&bmp->list);
597
598 bpl = (struct ulp_bde64 *)bmp->virt;
599 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
600 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
601 bpl->tus.f.bdeFlags = 0;
602 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
603 bpl->tus.w = le32_to_cpu(bpl->tus.w);
604 bpl++;
605
606 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
607 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
608 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
609 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
610 bpl->tus.w = le32_to_cpu(bpl->tus.w);
611
612 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
613 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
614 "rqstlen:%d rsplen:%d %pad %pad\n",
615 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
616 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
617 &pnvme_lsreq->rspdma);
618
619 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
620 pnvme_lsreq, gen_req_cmp, ndlp, 2,
621 pnvme_lsreq->timeout, 0);
622 if (ret != WQE_SUCCESS) {
623 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
624 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
625 "lsreq x%px Status %x DID %x\n",
626 pnvme_lsreq, ret, ndlp->nlp_DID);
627 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
628 kfree(bmp);
629 return -EIO;
630 }
631
632 return 0;
633}
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648static int
649lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
650 struct nvme_fc_remote_port *pnvme_rport,
651 struct nvmefc_ls_req *pnvme_lsreq)
652{
653 struct lpfc_nvme_lport *lport;
654 struct lpfc_nvme_rport *rport;
655 struct lpfc_vport *vport;
656 int ret;
657
658 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
659 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
660 if (unlikely(!lport) || unlikely(!rport))
661 return -EINVAL;
662
663 vport = lport->vport;
664 if (vport->load_flag & FC_UNLOADING)
665 return -ENODEV;
666
667 atomic_inc(&lport->fc4NvmeLsRequests);
668
669 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
670 lpfc_nvme_ls_req_cmp);
671 if (ret)
672 atomic_inc(&lport->xmt_ls_err);
673
674 return ret;
675}
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691int
692__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
693 struct nvmefc_ls_req *pnvme_lsreq)
694{
695 struct lpfc_hba *phba = vport->phba;
696 struct lpfc_sli_ring *pring;
697 struct lpfc_iocbq *wqe, *next_wqe;
698 bool foundit = false;
699
700 if (!ndlp) {
701 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
702 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
703 "x%06x, Failing LS Req\n",
704 ndlp, ndlp ? ndlp->nlp_DID : 0);
705 return -EINVAL;
706 }
707
708 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
709 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
710 "x%px rqstlen:%d rsplen:%d %pad %pad\n",
711 pnvme_lsreq, pnvme_lsreq->rqstlen,
712 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
713 &pnvme_lsreq->rspdma);
714
715
716
717
718
719 pring = phba->sli4_hba.nvmels_wq->pring;
720 spin_lock_irq(&phba->hbalock);
721 spin_lock(&pring->ring_lock);
722 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
723 if (wqe->context2 == pnvme_lsreq) {
724 wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
725 foundit = true;
726 break;
727 }
728 }
729 spin_unlock(&pring->ring_lock);
730
731 if (foundit)
732 lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
733 spin_unlock_irq(&phba->hbalock);
734
735 if (foundit)
736 return 0;
737
738 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
739 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
740 pnvme_lsreq);
741 return -EINVAL;
742}
743
744static int
745lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
746 struct nvme_fc_remote_port *remoteport,
747 struct nvmefc_ls_rsp *ls_rsp)
748{
749 struct lpfc_async_xchg_ctx *axchg =
750 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
751 struct lpfc_nvme_lport *lport;
752 int rc;
753
754 if (axchg->phba->pport->load_flag & FC_UNLOADING)
755 return -ENODEV;
756
757 lport = (struct lpfc_nvme_lport *)localport->private;
758
759 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
760
761 if (rc) {
762
763
764
765
766
767 if (rc != -EALREADY)
768 atomic_inc(&lport->xmt_ls_abort);
769 return rc;
770 }
771
772 return 0;
773}
774
775
776
777
778
779
780
781
782
783
784static void
785lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
786 struct nvme_fc_remote_port *pnvme_rport,
787 struct nvmefc_ls_req *pnvme_lsreq)
788{
789 struct lpfc_nvme_lport *lport;
790 struct lpfc_vport *vport;
791 struct lpfc_nodelist *ndlp;
792 int ret;
793
794 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
795 if (unlikely(!lport))
796 return;
797 vport = lport->vport;
798
799 if (vport->load_flag & FC_UNLOADING)
800 return;
801
802 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
803
804 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
805 if (!ret)
806 atomic_inc(&lport->xmt_ls_abort);
807}
808
809
810static inline void
811lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
812 struct lpfc_io_buf *lpfc_ncmd,
813 struct nvmefc_fcp_req *nCmd)
814{
815 struct lpfc_hba *phba = vport->phba;
816 struct sli4_sge *sgl;
817 union lpfc_wqe128 *wqe;
818 uint32_t *wptr, *dptr;
819
820
821
822
823
824
825
826 wqe = &lpfc_ncmd->cur_iocbq.wqe;
827
828
829
830
831
832
833
834
835 sgl = lpfc_ncmd->dma_sgl;
836 sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
837 if (phba->cfg_nvme_embed_cmd) {
838 sgl->addr_hi = 0;
839 sgl->addr_lo = 0;
840
841
842 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
843 wqe->generic.bde.tus.f.bdeSize = 56;
844 wqe->generic.bde.addrHigh = 0;
845 wqe->generic.bde.addrLow = 64;
846
847
848
849
850
851
852
853
854
855
856
857 wptr = &wqe->words[16];
858 dptr = (uint32_t *)nCmd->cmdaddr;
859 dptr++;
860
861 *wptr++ = *dptr++;
862 *wptr++ = *dptr++;
863 *wptr++ = *dptr++;
864 *wptr++ = *dptr++;
865 dptr++;
866 *wptr++ = *dptr++;
867 *wptr++ = *dptr++;
868 dptr += 8;
869 *wptr++ = *dptr++;
870 *wptr++ = *dptr++;
871 *wptr++ = *dptr++;
872 *wptr++ = *dptr++;
873 *wptr++ = *dptr++;
874 *wptr++ = *dptr++;
875 *wptr++ = *dptr++;
876 *wptr = *dptr;
877 } else {
878 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
879 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
880
881
882 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
883 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
884 wqe->generic.bde.addrHigh = sgl->addr_hi;
885 wqe->generic.bde.addrLow = sgl->addr_lo;
886
887
888 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
889 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
890 }
891
892 sgl++;
893
894
895 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
896 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
897 sgl->word2 = le32_to_cpu(sgl->word2);
898 if (nCmd->sg_cnt)
899 bf_set(lpfc_sli4_sge_last, sgl, 0);
900 else
901 bf_set(lpfc_sli4_sge_last, sgl, 1);
902 sgl->word2 = cpu_to_le32(sgl->word2);
903 sgl->sge_len = cpu_to_le32(nCmd->rsplen);
904}
905
906
907
908
909
910
911
912
913
914
915
916
917
918static void
919lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
920 struct lpfc_wcqe_complete *wcqe)
921{
922 struct lpfc_io_buf *lpfc_ncmd =
923 (struct lpfc_io_buf *)pwqeIn->context1;
924 struct lpfc_vport *vport = pwqeIn->vport;
925 struct nvmefc_fcp_req *nCmd;
926 struct nvme_fc_ersp_iu *ep;
927 struct nvme_fc_cmd_iu *cp;
928 struct lpfc_nodelist *ndlp;
929 struct lpfc_nvme_fcpreq_priv *freqpriv;
930 struct lpfc_nvme_lport *lport;
931 uint32_t code, status, idx;
932 uint16_t cid, sqhd, data;
933 uint32_t *ptr;
934 uint32_t lat;
935 bool call_done = false;
936#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
937 int cpu;
938#endif
939
940
941 if (!lpfc_ncmd) {
942 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
943 "6071 Null lpfc_ncmd pointer. No "
944 "release, skip completion\n");
945 return;
946 }
947
948
949 spin_lock(&lpfc_ncmd->buf_lock);
950
951 if (!lpfc_ncmd->nvmeCmd) {
952 spin_unlock(&lpfc_ncmd->buf_lock);
953 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
954 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
955 "nvmeCmd x%px\n",
956 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
957
958
959 lpfc_release_nvme_buf(phba, lpfc_ncmd);
960 return;
961 }
962 nCmd = lpfc_ncmd->nvmeCmd;
963 status = bf_get(lpfc_wcqe_c_status, wcqe);
964
965 idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
966 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
967
968 if (unlikely(status && vport->localport)) {
969 lport = (struct lpfc_nvme_lport *)vport->localport->private;
970 if (lport) {
971 if (bf_get(lpfc_wcqe_c_xb, wcqe))
972 atomic_inc(&lport->cmpl_fcp_xb);
973 atomic_inc(&lport->cmpl_fcp_err);
974 }
975 }
976
977 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
978 lpfc_ncmd->cur_iocbq.sli4_xritag,
979 status, wcqe->parameter);
980
981
982
983
984 ndlp = lpfc_ncmd->ndlp;
985 if (!ndlp) {
986 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
987 "6062 Ignoring NVME cmpl. No ndlp\n");
988 goto out_err;
989 }
990
991 code = bf_get(lpfc_wcqe_c_code, wcqe);
992 if (code == CQE_CODE_NVME_ERSP) {
993
994 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
995
996
997
998
999
1000 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1001 cid = cp->sqe.common.command_id;
1002
1003
1004
1005
1006
1007
1008
1009 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1010
1011
1012 ep->iu_len = cpu_to_be16(8);
1013 ep->rsn = wcqe->parameter;
1014 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1015 ep->rsvd12 = 0;
1016 ptr = (uint32_t *)&ep->cqe.result.u64;
1017 *ptr++ = wcqe->total_data_placed;
1018 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1019 *ptr = (uint32_t)data;
1020 ep->cqe.sq_head = sqhd;
1021 ep->cqe.sq_id = nCmd->sqid;
1022 ep->cqe.command_id = cid;
1023 ep->cqe.status = 0;
1024
1025 lpfc_ncmd->status = IOSTAT_SUCCESS;
1026 lpfc_ncmd->result = 0;
1027 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1028 nCmd->transferred_length = nCmd->payload_length;
1029 } else {
1030 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1031 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044 switch (lpfc_ncmd->status) {
1045 case IOSTAT_SUCCESS:
1046 nCmd->transferred_length = wcqe->total_data_placed;
1047 nCmd->rcv_rsplen = 0;
1048 nCmd->status = 0;
1049 break;
1050 case IOSTAT_FCP_RSP_ERROR:
1051 nCmd->transferred_length = wcqe->total_data_placed;
1052 nCmd->rcv_rsplen = wcqe->parameter;
1053 nCmd->status = 0;
1054
1055
1056 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1057 lpfc_ncmd->status = IOSTAT_SUCCESS;
1058 lpfc_ncmd->result = 0;
1059
1060 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1061 "6084 NVME Completion ERSP: "
1062 "xri %x placed x%x\n",
1063 lpfc_ncmd->cur_iocbq.sli4_xritag,
1064 wcqe->total_data_placed);
1065 break;
1066 }
1067 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1068 "6081 NVME Completion Protocol Error: "
1069 "xri %x status x%x result x%x "
1070 "placed x%x\n",
1071 lpfc_ncmd->cur_iocbq.sli4_xritag,
1072 lpfc_ncmd->status, lpfc_ncmd->result,
1073 wcqe->total_data_placed);
1074 break;
1075 case IOSTAT_LOCAL_REJECT:
1076
1077 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1078 lpfc_printf_vlog(vport, KERN_INFO,
1079 LOG_NVME_IOERR,
1080 "6032 Delay Aborted cmd x%px "
1081 "nvme cmd x%px, xri x%x, "
1082 "xb %d\n",
1083 lpfc_ncmd, nCmd,
1084 lpfc_ncmd->cur_iocbq.sli4_xritag,
1085 bf_get(lpfc_wcqe_c_xb, wcqe));
1086 fallthrough;
1087 default:
1088out_err:
1089 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1090 "6072 NVME Completion Error: xri %x "
1091 "status x%x result x%x [x%x] "
1092 "placed x%x\n",
1093 lpfc_ncmd->cur_iocbq.sli4_xritag,
1094 lpfc_ncmd->status, lpfc_ncmd->result,
1095 wcqe->parameter,
1096 wcqe->total_data_placed);
1097 nCmd->transferred_length = 0;
1098 nCmd->rcv_rsplen = 0;
1099 nCmd->status = NVME_SC_INTERNAL;
1100 }
1101 }
1102
1103
1104 if (bf_get(lpfc_wcqe_c_xb, wcqe))
1105 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1106 else
1107 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1108
1109
1110
1111
1112
1113#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1114 if (lpfc_ncmd->ts_cmd_start) {
1115 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1116 lpfc_ncmd->ts_data_io = ktime_get_ns();
1117 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1118 lpfc_io_ktime(phba, lpfc_ncmd);
1119 }
1120 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1121 cpu = raw_smp_processor_id();
1122 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1123 if (lpfc_ncmd->cpu != cpu)
1124 lpfc_printf_vlog(vport,
1125 KERN_INFO, LOG_NVME_IOERR,
1126 "6701 CPU Check cmpl: "
1127 "cpu %d expect %d\n",
1128 cpu, lpfc_ncmd->cpu);
1129 }
1130#endif
1131
1132
1133
1134
1135
1136 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1137 freqpriv = nCmd->private;
1138 freqpriv->nvme_buf = NULL;
1139 lpfc_ncmd->nvmeCmd = NULL;
1140 call_done = true;
1141 }
1142 spin_unlock(&lpfc_ncmd->buf_lock);
1143
1144
1145 if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1146 nCmd->io_dir == NVMEFC_FCP_READ &&
1147 nCmd->payload_length) {
1148
1149 lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
1150 lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);
1151 }
1152
1153 if (call_done)
1154 nCmd->done(nCmd);
1155
1156
1157 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1158}
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176static int
1177lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1178 struct lpfc_io_buf *lpfc_ncmd,
1179 struct lpfc_nodelist *pnode,
1180 struct lpfc_fc4_ctrl_stat *cstat)
1181{
1182 struct lpfc_hba *phba = vport->phba;
1183 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1184 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1185 union lpfc_wqe128 *wqe = &pwqeq->wqe;
1186 uint32_t req_len;
1187
1188
1189
1190
1191
1192 if (nCmd->sg_cnt) {
1193 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1194
1195 memcpy(&wqe->words[7],
1196 &lpfc_iwrite_cmd_template.words[7],
1197 sizeof(uint32_t) * 5);
1198
1199
1200 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1201
1202
1203 if ((phba->cfg_nvme_enable_fb) &&
1204 (pnode->nlp_flag & NLP_FIRSTBURST)) {
1205 req_len = lpfc_ncmd->nvmeCmd->payload_length;
1206 if (req_len < pnode->nvme_fb_size)
1207 wqe->fcp_iwrite.initial_xfer_len =
1208 req_len;
1209 else
1210 wqe->fcp_iwrite.initial_xfer_len =
1211 pnode->nvme_fb_size;
1212 } else {
1213 wqe->fcp_iwrite.initial_xfer_len = 0;
1214 }
1215 cstat->output_requests++;
1216 } else {
1217
1218 memcpy(&wqe->words[7],
1219 &lpfc_iread_cmd_template.words[7],
1220 sizeof(uint32_t) * 5);
1221
1222
1223 wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1224
1225
1226 wqe->fcp_iread.rsrvd5 = 0;
1227
1228
1229 if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
1230 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
1231 LPFC_WQE_IOD_NONE);
1232 cstat->input_requests++;
1233 }
1234 } else {
1235
1236 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1237 sizeof(uint32_t) * 8);
1238 cstat->control_requests++;
1239 }
1240
1241 if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1242 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1243
1244
1245
1246
1247
1248
1249 bf_set(payload_offset_len, &wqe->fcp_icmd,
1250 (nCmd->rsplen + nCmd->cmdlen));
1251
1252
1253 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1254 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1255 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1256
1257
1258 wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1259
1260
1261 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1262
1263
1264 bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1265
1266
1267
1268 pwqeq->vport = vport;
1269 return 0;
1270}
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286static int
1287lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1288 struct lpfc_io_buf *lpfc_ncmd)
1289{
1290 struct lpfc_hba *phba = vport->phba;
1291 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1292 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1293 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1294 struct sli4_hybrid_sgl *sgl_xtra = NULL;
1295 struct scatterlist *data_sg;
1296 struct sli4_sge *first_data_sgl;
1297 struct ulp_bde64 *bde;
1298 dma_addr_t physaddr = 0;
1299 uint32_t num_bde = 0;
1300 uint32_t dma_len = 0;
1301 uint32_t dma_offset = 0;
1302 int nseg, i, j;
1303 bool lsp_just_set = false;
1304
1305
1306 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1307
1308
1309
1310
1311
1312 if (nCmd->sg_cnt) {
1313
1314
1315
1316
1317 sgl += 2;
1318
1319 first_data_sgl = sgl;
1320 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1321 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1322 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1323 "6058 Too many sg segments from "
1324 "NVME Transport. Max %d, "
1325 "nvmeIO sg_cnt %d\n",
1326 phba->cfg_nvme_seg_cnt + 1,
1327 lpfc_ncmd->seg_cnt);
1328 lpfc_ncmd->seg_cnt = 0;
1329 return 1;
1330 }
1331
1332
1333
1334
1335
1336
1337
1338 nseg = nCmd->sg_cnt;
1339 data_sg = nCmd->first_sgl;
1340
1341
1342 j = 2;
1343 for (i = 0; i < nseg; i++) {
1344 if (data_sg == NULL) {
1345 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1346 "6059 dptr err %d, nseg %d\n",
1347 i, nseg);
1348 lpfc_ncmd->seg_cnt = 0;
1349 return 1;
1350 }
1351
1352 sgl->word2 = 0;
1353 if ((num_bde + 1) == nseg) {
1354 bf_set(lpfc_sli4_sge_last, sgl, 1);
1355 bf_set(lpfc_sli4_sge_type, sgl,
1356 LPFC_SGE_TYPE_DATA);
1357 } else {
1358 bf_set(lpfc_sli4_sge_last, sgl, 0);
1359
1360
1361 if (!lsp_just_set &&
1362 !((j + 1) % phba->border_sge_num) &&
1363 ((nseg - 1) != i)) {
1364
1365 bf_set(lpfc_sli4_sge_type, sgl,
1366 LPFC_SGE_TYPE_LSP);
1367
1368 sgl_xtra = lpfc_get_sgl_per_hdwq(
1369 phba, lpfc_ncmd);
1370
1371 if (unlikely(!sgl_xtra)) {
1372 lpfc_ncmd->seg_cnt = 0;
1373 return 1;
1374 }
1375 sgl->addr_lo = cpu_to_le32(putPaddrLow(
1376 sgl_xtra->dma_phys_sgl));
1377 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1378 sgl_xtra->dma_phys_sgl));
1379
1380 } else {
1381 bf_set(lpfc_sli4_sge_type, sgl,
1382 LPFC_SGE_TYPE_DATA);
1383 }
1384 }
1385
1386 if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1387 LPFC_SGE_TYPE_LSP)) {
1388 if ((nseg - 1) == i)
1389 bf_set(lpfc_sli4_sge_last, sgl, 1);
1390
1391 physaddr = data_sg->dma_address;
1392 dma_len = data_sg->length;
1393 sgl->addr_lo = cpu_to_le32(
1394 putPaddrLow(physaddr));
1395 sgl->addr_hi = cpu_to_le32(
1396 putPaddrHigh(physaddr));
1397
1398 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1399 sgl->word2 = cpu_to_le32(sgl->word2);
1400 sgl->sge_len = cpu_to_le32(dma_len);
1401
1402 dma_offset += dma_len;
1403 data_sg = sg_next(data_sg);
1404
1405 sgl++;
1406
1407 lsp_just_set = false;
1408 } else {
1409 sgl->word2 = cpu_to_le32(sgl->word2);
1410
1411 sgl->sge_len = cpu_to_le32(
1412 phba->cfg_sg_dma_buf_size);
1413
1414 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1415 i = i - 1;
1416
1417 lsp_just_set = true;
1418 }
1419
1420 j++;
1421 }
1422 if (phba->cfg_enable_pbde) {
1423
1424
1425 bde = (struct ulp_bde64 *)
1426 &wqe->words[13];
1427 bde->addrLow = first_data_sgl->addr_lo;
1428 bde->addrHigh = first_data_sgl->addr_hi;
1429 bde->tus.f.bdeSize =
1430 le32_to_cpu(first_data_sgl->sge_len);
1431 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1432 bde->tus.w = cpu_to_le32(bde->tus.w);
1433
1434
1435 bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
1436 } else {
1437 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1438 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1439 }
1440
1441 } else {
1442 lpfc_ncmd->seg_cnt = 0;
1443
1444
1445
1446
1447 if (nCmd->payload_length != 0) {
1448 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1449 "6063 NVME DMA Prep Err: sg_cnt %d "
1450 "payload_length x%x\n",
1451 nCmd->sg_cnt, nCmd->payload_length);
1452 return 1;
1453 }
1454 }
1455 return 0;
1456}
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473static int
1474lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1475 struct nvme_fc_remote_port *pnvme_rport,
1476 void *hw_queue_handle,
1477 struct nvmefc_fcp_req *pnvme_fcreq)
1478{
1479 int ret = 0;
1480 int expedite = 0;
1481 int idx, cpu;
1482 struct lpfc_nvme_lport *lport;
1483 struct lpfc_fc4_ctrl_stat *cstat;
1484 struct lpfc_vport *vport;
1485 struct lpfc_hba *phba;
1486 struct lpfc_nodelist *ndlp;
1487 struct lpfc_io_buf *lpfc_ncmd;
1488 struct lpfc_nvme_rport *rport;
1489 struct lpfc_nvme_qhandle *lpfc_queue_info;
1490 struct lpfc_nvme_fcpreq_priv *freqpriv;
1491 struct nvme_common_command *sqe;
1492 uint64_t start = 0;
1493
1494
1495
1496
1497 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1498 if (unlikely(!lport)) {
1499 ret = -EINVAL;
1500 goto out_fail;
1501 }
1502
1503 vport = lport->vport;
1504
1505 if (unlikely(!hw_queue_handle)) {
1506 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1507 "6117 Fail IO, NULL hw_queue_handle\n");
1508 atomic_inc(&lport->xmt_fcp_err);
1509 ret = -EBUSY;
1510 goto out_fail;
1511 }
1512
1513 phba = vport->phba;
1514
1515 if (unlikely(vport->load_flag & FC_UNLOADING)) {
1516 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1517 "6124 Fail IO, Driver unload\n");
1518 atomic_inc(&lport->xmt_fcp_err);
1519 ret = -ENODEV;
1520 goto out_fail;
1521 }
1522
1523 freqpriv = pnvme_fcreq->private;
1524 if (unlikely(!freqpriv)) {
1525 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1526 "6158 Fail IO, NULL request data\n");
1527 atomic_inc(&lport->xmt_fcp_err);
1528 ret = -EINVAL;
1529 goto out_fail;
1530 }
1531
1532#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1533 if (phba->ktime_on)
1534 start = ktime_get_ns();
1535#endif
1536 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1537 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1538
1539
1540
1541
1542
1543 ndlp = rport->ndlp;
1544 if (!ndlp) {
1545 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1546 "6053 Busy IO, ndlp not ready: rport x%px "
1547 "ndlp x%px, DID x%06x\n",
1548 rport, ndlp, pnvme_rport->port_id);
1549 atomic_inc(&lport->xmt_fcp_err);
1550 ret = -EBUSY;
1551 goto out_fail;
1552 }
1553
1554
1555 if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1556 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1557 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1558 "6036 Fail IO, DID x%06x not ready for "
1559 "IO. State x%x, Type x%x Flg x%x\n",
1560 pnvme_rport->port_id,
1561 ndlp->nlp_state, ndlp->nlp_type,
1562 ndlp->fc4_xpt_flags);
1563 atomic_inc(&lport->xmt_fcp_bad_ndlp);
1564 ret = -EBUSY;
1565 goto out_fail;
1566
1567 }
1568
1569
1570
1571
1572
1573 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1574 sqe = &((struct nvme_fc_cmd_iu *)
1575 pnvme_fcreq->cmdaddr)->sqe.common;
1576 if (sqe->opcode == nvme_admin_keep_alive)
1577 expedite = 1;
1578 }
1579
1580
1581 if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1582 pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
1583 pnvme_fcreq->payload_length) {
1584 ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);
1585 if (ret) {
1586 ret = -EBUSY;
1587 goto out_fail;
1588 }
1589
1590 start = ktime_get_ns();
1591 }
1592
1593
1594
1595
1596 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1597 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1598 !expedite) {
1599 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1600 "6174 Fail IO, ndlp qdepth exceeded: "
1601 "idx %d DID %x pend %d qdepth %d\n",
1602 lpfc_queue_info->index, ndlp->nlp_DID,
1603 atomic_read(&ndlp->cmd_pending),
1604 ndlp->cmd_qdepth);
1605 atomic_inc(&lport->xmt_fcp_qdepth);
1606 ret = -EBUSY;
1607 goto out_fail1;
1608 }
1609 }
1610
1611
1612 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1613 idx = lpfc_queue_info->index;
1614 } else {
1615 cpu = raw_smp_processor_id();
1616 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1617 }
1618
1619 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1620 if (lpfc_ncmd == NULL) {
1621 atomic_inc(&lport->xmt_fcp_noxri);
1622 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1623 "6065 Fail IO, driver buffer pool is empty: "
1624 "idx %d DID %x\n",
1625 lpfc_queue_info->index, ndlp->nlp_DID);
1626 ret = -EBUSY;
1627 goto out_fail1;
1628 }
1629#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1630 if (start) {
1631 lpfc_ncmd->ts_cmd_start = start;
1632 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1633 } else {
1634 lpfc_ncmd->ts_cmd_start = 0;
1635 }
1636#endif
1637 lpfc_ncmd->rx_cmd_start = start;
1638
1639
1640
1641
1642
1643
1644
1645 freqpriv->nvme_buf = lpfc_ncmd;
1646 lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1647 lpfc_ncmd->ndlp = ndlp;
1648 lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658 lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1659 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1660
1661 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1662 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1663 if (ret) {
1664 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1665 "6175 Fail IO, Prep DMA: "
1666 "idx %d DID %x\n",
1667 lpfc_queue_info->index, ndlp->nlp_DID);
1668 atomic_inc(&lport->xmt_fcp_err);
1669 ret = -ENOMEM;
1670 goto out_free_nvme_buf;
1671 }
1672
1673 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1674 lpfc_ncmd->cur_iocbq.sli4_xritag,
1675 lpfc_queue_info->index, ndlp->nlp_DID);
1676
1677 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1678 if (ret) {
1679 atomic_inc(&lport->xmt_fcp_wqerr);
1680 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1681 "6113 Fail IO, Could not issue WQE err %x "
1682 "sid: x%x did: x%x oxid: x%x\n",
1683 ret, vport->fc_myDID, ndlp->nlp_DID,
1684 lpfc_ncmd->cur_iocbq.sli4_xritag);
1685 goto out_free_nvme_buf;
1686 }
1687
1688 if (phba->cfg_xri_rebalancing)
1689 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1690
1691#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1692 if (lpfc_ncmd->ts_cmd_start)
1693 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1694
1695 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1696 cpu = raw_smp_processor_id();
1697 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1698 lpfc_ncmd->cpu = cpu;
1699 if (idx != cpu)
1700 lpfc_printf_vlog(vport,
1701 KERN_INFO, LOG_NVME_IOERR,
1702 "6702 CPU Check cmd: "
1703 "cpu %d wq %d\n",
1704 lpfc_ncmd->cpu,
1705 lpfc_queue_info->index);
1706 }
1707#endif
1708 return 0;
1709
1710 out_free_nvme_buf:
1711 if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1712 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1713 cstat->output_requests--;
1714 else
1715 cstat->input_requests--;
1716 } else
1717 cstat->control_requests--;
1718 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1719 out_fail1:
1720 lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
1721 pnvme_fcreq->payload_length, NULL);
1722 out_fail:
1723 return ret;
1724}
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737void
1738lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1739 struct lpfc_wcqe_complete *abts_cmpl)
1740{
1741 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1742 "6145 ABORT_XRI_CN completing on rpi x%x "
1743 "original iotag x%x, abort cmd iotag x%x "
1744 "req_tag x%x, status x%x, hwstatus x%x\n",
1745 cmdiocb->iocb.un.acxri.abortContextTag,
1746 cmdiocb->iocb.un.acxri.abortIoTag,
1747 cmdiocb->iotag,
1748 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1749 bf_get(lpfc_wcqe_c_status, abts_cmpl),
1750 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1751 lpfc_sli_release_iocbq(phba, cmdiocb);
1752}
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770static void
1771lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1772 struct nvme_fc_remote_port *pnvme_rport,
1773 void *hw_queue_handle,
1774 struct nvmefc_fcp_req *pnvme_fcreq)
1775{
1776 struct lpfc_nvme_lport *lport;
1777 struct lpfc_vport *vport;
1778 struct lpfc_hba *phba;
1779 struct lpfc_io_buf *lpfc_nbuf;
1780 struct lpfc_iocbq *nvmereq_wqe;
1781 struct lpfc_nvme_fcpreq_priv *freqpriv;
1782 unsigned long flags;
1783 int ret_val;
1784
1785
1786
1787
1788 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1789 if (unlikely(!lport))
1790 return;
1791
1792 vport = lport->vport;
1793
1794 if (unlikely(!hw_queue_handle)) {
1795 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1796 "6129 Fail Abort, HW Queue Handle NULL.\n");
1797 return;
1798 }
1799
1800 phba = vport->phba;
1801 freqpriv = pnvme_fcreq->private;
1802
1803 if (unlikely(!freqpriv))
1804 return;
1805 if (vport->load_flag & FC_UNLOADING)
1806 return;
1807
1808
1809 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1810 "6002 Abort Request to rport DID x%06x "
1811 "for nvme_fc_req x%px\n",
1812 pnvme_rport->port_id,
1813 pnvme_fcreq);
1814
1815
1816
1817
1818 spin_lock_irqsave(&phba->hbalock, flags);
1819
1820 if (phba->hba_flag & HBA_IOQ_FLUSH) {
1821 spin_unlock_irqrestore(&phba->hbalock, flags);
1822 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1823 "6139 Driver in reset cleanup - flushing "
1824 "NVME Req now. hba_flag x%x\n",
1825 phba->hba_flag);
1826 return;
1827 }
1828
1829 lpfc_nbuf = freqpriv->nvme_buf;
1830 if (!lpfc_nbuf) {
1831 spin_unlock_irqrestore(&phba->hbalock, flags);
1832 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1833 "6140 NVME IO req has no matching lpfc nvme "
1834 "io buffer. Skipping abort req.\n");
1835 return;
1836 } else if (!lpfc_nbuf->nvmeCmd) {
1837 spin_unlock_irqrestore(&phba->hbalock, flags);
1838 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1839 "6141 lpfc NVME IO req has no nvme_fcreq "
1840 "io buffer. Skipping abort req.\n");
1841 return;
1842 }
1843 nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1844
1845
1846 spin_lock(&lpfc_nbuf->buf_lock);
1847
1848
1849
1850
1851
1852
1853
1854
1855 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1856 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1857 "6143 NVME req mismatch: "
1858 "lpfc_nbuf x%px nvmeCmd x%px, "
1859 "pnvme_fcreq x%px. Skipping Abort xri x%x\n",
1860 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1861 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1862 goto out_unlock;
1863 }
1864
1865
1866 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1867 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1868 "6142 NVME IO req x%px not queued - skipping "
1869 "abort req xri x%x\n",
1870 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1871 goto out_unlock;
1872 }
1873
1874 atomic_inc(&lport->xmt_fcp_abort);
1875 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1876 nvmereq_wqe->sli4_xritag,
1877 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1878
1879
1880 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
1881 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1882 "6144 Outstanding NVME I/O Abort Request "
1883 "still pending on nvme_fcreq x%px, "
1884 "lpfc_ncmd x%px xri x%x\n",
1885 pnvme_fcreq, lpfc_nbuf,
1886 nvmereq_wqe->sli4_xritag);
1887 goto out_unlock;
1888 }
1889
1890 ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,
1891 lpfc_nvme_abort_fcreq_cmpl);
1892
1893 spin_unlock(&lpfc_nbuf->buf_lock);
1894 spin_unlock_irqrestore(&phba->hbalock, flags);
1895
1896
1897 lpfc_issue_hb_tmo(phba);
1898
1899 if (ret_val != WQE_SUCCESS) {
1900 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1901 "6137 Failed abts issue_wqe with status x%x "
1902 "for nvme_fcreq x%px.\n",
1903 ret_val, pnvme_fcreq);
1904 return;
1905 }
1906
1907 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1908 "6138 Transport Abort NVME Request Issued for "
1909 "ox_id x%x\n",
1910 nvmereq_wqe->sli4_xritag);
1911 return;
1912
1913out_unlock:
1914 spin_unlock(&lpfc_nbuf->buf_lock);
1915 spin_unlock_irqrestore(&phba->hbalock, flags);
1916 return;
1917}
1918
1919
1920static struct nvme_fc_port_template lpfc_nvme_template = {
1921
1922 .localport_delete = lpfc_nvme_localport_delete,
1923 .remoteport_delete = lpfc_nvme_remoteport_delete,
1924 .create_queue = lpfc_nvme_create_queue,
1925 .delete_queue = lpfc_nvme_delete_queue,
1926 .ls_req = lpfc_nvme_ls_req,
1927 .fcp_io = lpfc_nvme_fcp_io_submit,
1928 .ls_abort = lpfc_nvme_ls_abort,
1929 .fcp_abort = lpfc_nvme_fcp_abort,
1930 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
1931
1932 .max_hw_queues = 1,
1933 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
1934 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
1935 .dma_boundary = 0xFFFFFFFF,
1936
1937
1938
1939
1940 .local_priv_sz = sizeof(struct lpfc_nvme_lport),
1941 .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
1942 .lsrqst_priv_sz = 0,
1943 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
1944};
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956static struct lpfc_io_buf *
1957lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
1958 int idx, int expedite)
1959{
1960 struct lpfc_io_buf *lpfc_ncmd;
1961 struct lpfc_sli4_hdw_queue *qp;
1962 struct sli4_sge *sgl;
1963 struct lpfc_iocbq *pwqeq;
1964 union lpfc_wqe128 *wqe;
1965
1966 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
1967
1968 if (lpfc_ncmd) {
1969 pwqeq = &(lpfc_ncmd->cur_iocbq);
1970 wqe = &pwqeq->wqe;
1971
1972
1973
1974
1975 pwqeq->iocb_flag = LPFC_IO_NVME;
1976 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
1977 lpfc_ncmd->start_time = jiffies;
1978 lpfc_ncmd->flags = 0;
1979
1980
1981
1982
1983
1984 sgl = lpfc_ncmd->dma_sgl;
1985 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1986 bf_set(lpfc_sli4_sge_last, sgl, 0);
1987 sgl->word2 = cpu_to_le32(sgl->word2);
1988
1989
1990
1991 memset(wqe, 0, sizeof(union lpfc_wqe));
1992
1993 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1994 atomic_inc(&ndlp->cmd_pending);
1995 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
1996 }
1997
1998 } else {
1999 qp = &phba->sli4_hba.hdwq[idx];
2000 qp->empty_io_bufs++;
2001 }
2002
2003 return lpfc_ncmd;
2004}
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016static void
2017lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2018{
2019 struct lpfc_sli4_hdw_queue *qp;
2020 unsigned long iflag = 0;
2021
2022 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2023 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2024
2025 lpfc_ncmd->ndlp = NULL;
2026 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2027
2028 qp = lpfc_ncmd->hdwq;
2029 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2030 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2031 "6310 XB release deferred for "
2032 "ox_id x%x on reqtag x%x\n",
2033 lpfc_ncmd->cur_iocbq.sli4_xritag,
2034 lpfc_ncmd->cur_iocbq.iotag);
2035
2036 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2037 list_add_tail(&lpfc_ncmd->list,
2038 &qp->lpfc_abts_io_buf_list);
2039 qp->abts_nvme_io_bufs++;
2040 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2041 } else
2042 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2043}
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061int
2062lpfc_nvme_create_localport(struct lpfc_vport *vport)
2063{
2064 int ret = 0;
2065 struct lpfc_hba *phba = vport->phba;
2066 struct nvme_fc_port_info nfcp_info;
2067 struct nvme_fc_local_port *localport;
2068 struct lpfc_nvme_lport *lport;
2069
2070
2071
2072
2073 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2074 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2075 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2076 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2077
2078
2079
2080
2081
2082 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2083
2084
2085
2086
2087 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2088
2089 if (!IS_ENABLED(CONFIG_NVME_FC))
2090 return ret;
2091
2092
2093
2094
2095
2096 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2097 &vport->phba->pcidev->dev, &localport);
2098 if (!ret) {
2099 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2100 "6005 Successfully registered local "
2101 "NVME port num %d, localP x%px, private "
2102 "x%px, sg_seg %d\n",
2103 localport->port_num, localport,
2104 localport->private,
2105 lpfc_nvme_template.max_sgl_segments);
2106
2107
2108 lport = (struct lpfc_nvme_lport *)localport->private;
2109 vport->localport = localport;
2110 lport->vport = vport;
2111 vport->nvmei_support = 1;
2112
2113 atomic_set(&lport->xmt_fcp_noxri, 0);
2114 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2115 atomic_set(&lport->xmt_fcp_qdepth, 0);
2116 atomic_set(&lport->xmt_fcp_err, 0);
2117 atomic_set(&lport->xmt_fcp_wqerr, 0);
2118 atomic_set(&lport->xmt_fcp_abort, 0);
2119 atomic_set(&lport->xmt_ls_abort, 0);
2120 atomic_set(&lport->xmt_ls_err, 0);
2121 atomic_set(&lport->cmpl_fcp_xb, 0);
2122 atomic_set(&lport->cmpl_fcp_err, 0);
2123 atomic_set(&lport->cmpl_ls_xb, 0);
2124 atomic_set(&lport->cmpl_ls_err, 0);
2125
2126 atomic_set(&lport->fc4NvmeLsRequests, 0);
2127 atomic_set(&lport->fc4NvmeLsCmpls, 0);
2128 }
2129
2130 return ret;
2131}
2132
2133#if (IS_ENABLED(CONFIG_NVME_FC))
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144static void
2145lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2146 struct lpfc_nvme_lport *lport,
2147 struct completion *lport_unreg_cmp)
2148{
2149 u32 wait_tmo;
2150 int ret, i, pending = 0;
2151 struct lpfc_sli_ring *pring;
2152 struct lpfc_hba *phba = vport->phba;
2153 struct lpfc_sli4_hdw_queue *qp;
2154 int abts_scsi, abts_nvme;
2155
2156
2157
2158
2159
2160 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2161 while (true) {
2162 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2163 if (unlikely(!ret)) {
2164 pending = 0;
2165 abts_scsi = 0;
2166 abts_nvme = 0;
2167 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2168 qp = &phba->sli4_hba.hdwq[i];
2169 pring = qp->io_wq->pring;
2170 if (!pring)
2171 continue;
2172 pending += pring->txcmplq_cnt;
2173 abts_scsi += qp->abts_scsi_io_bufs;
2174 abts_nvme += qp->abts_nvme_io_bufs;
2175 }
2176 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2177 "6176 Lport x%px Localport x%px wait "
2178 "timed out. Pending %d [%d:%d]. "
2179 "Renewing.\n",
2180 lport, vport->localport, pending,
2181 abts_scsi, abts_nvme);
2182 continue;
2183 }
2184 break;
2185 }
2186 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2187 "6177 Lport x%px Localport x%px Complete Success\n",
2188 lport, vport->localport);
2189}
2190#endif
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202void
2203lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2204{
2205#if (IS_ENABLED(CONFIG_NVME_FC))
2206 struct nvme_fc_local_port *localport;
2207 struct lpfc_nvme_lport *lport;
2208 int ret;
2209 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2210
2211 if (vport->nvmei_support == 0)
2212 return;
2213
2214 localport = vport->localport;
2215 lport = (struct lpfc_nvme_lport *)localport->private;
2216
2217 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2218 "6011 Destroying NVME localport x%px\n",
2219 localport);
2220
2221
2222
2223
2224 lport->lport_unreg_cmp = &lport_unreg_cmp;
2225 ret = nvme_fc_unregister_localport(localport);
2226
2227
2228
2229
2230 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2231 vport->localport = NULL;
2232
2233
2234
2235
2236
2237 vport->nvmei_support = 0;
2238 if (ret == 0) {
2239 lpfc_printf_vlog(vport,
2240 KERN_INFO, LOG_NVME_DISC,
2241 "6009 Unregistered lport Success\n");
2242 } else {
2243 lpfc_printf_vlog(vport,
2244 KERN_INFO, LOG_NVME_DISC,
2245 "6010 Unregistered lport "
2246 "Failed, status x%x\n",
2247 ret);
2248 }
2249#endif
2250}
2251
2252void
2253lpfc_nvme_update_localport(struct lpfc_vport *vport)
2254{
2255#if (IS_ENABLED(CONFIG_NVME_FC))
2256 struct nvme_fc_local_port *localport;
2257 struct lpfc_nvme_lport *lport;
2258
2259 localport = vport->localport;
2260 if (!localport) {
2261 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2262 "6710 Update NVME fail. No localport\n");
2263 return;
2264 }
2265 lport = (struct lpfc_nvme_lport *)localport->private;
2266 if (!lport) {
2267 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2268 "6171 Update NVME fail. localP x%px, No lport\n",
2269 localport);
2270 return;
2271 }
2272 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2273 "6012 Update NVME lport x%px did x%x\n",
2274 localport, vport->fc_myDID);
2275
2276 localport->port_id = vport->fc_myDID;
2277 if (localport->port_id == 0)
2278 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2279 else
2280 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2281
2282 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2283 "6030 bound lport x%px to DID x%06x\n",
2284 lport, localport->port_id);
2285#endif
2286}
2287
2288int
2289lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2290{
2291#if (IS_ENABLED(CONFIG_NVME_FC))
2292 int ret = 0;
2293 struct nvme_fc_local_port *localport;
2294 struct lpfc_nvme_lport *lport;
2295 struct lpfc_nvme_rport *rport;
2296 struct lpfc_nvme_rport *oldrport;
2297 struct nvme_fc_remote_port *remote_port;
2298 struct nvme_fc_port_info rpinfo;
2299 struct lpfc_nodelist *prev_ndlp = NULL;
2300 struct fc_rport *srport = ndlp->rport;
2301
2302 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2303 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2304 ndlp->nlp_DID, ndlp->nlp_type);
2305
2306 localport = vport->localport;
2307 if (!localport)
2308 return 0;
2309
2310 lport = (struct lpfc_nvme_lport *)localport->private;
2311
2312
2313
2314
2315
2316
2317
2318 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2319 rpinfo.port_id = ndlp->nlp_DID;
2320 if (ndlp->nlp_type & NLP_NVME_TARGET)
2321 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2322 if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2323 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2324
2325 if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2326 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2327
2328 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2329 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2330 if (srport)
2331 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2332 else
2333 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2334
2335 spin_lock_irq(&ndlp->lock);
2336 oldrport = lpfc_ndlp_get_nrport(ndlp);
2337 if (oldrport) {
2338 prev_ndlp = oldrport->ndlp;
2339 spin_unlock_irq(&ndlp->lock);
2340 } else {
2341 spin_unlock_irq(&ndlp->lock);
2342 if (!lpfc_nlp_get(ndlp)) {
2343 dev_warn(&vport->phba->pcidev->dev,
2344 "Warning - No node ref - exit register\n");
2345 return 0;
2346 }
2347 }
2348
2349 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2350 if (!ret) {
2351
2352
2353
2354
2355
2356
2357
2358 spin_lock_irq(&ndlp->lock);
2359 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2360 ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2361 spin_unlock_irq(&ndlp->lock);
2362 rport = remote_port->private;
2363 if (oldrport) {
2364
2365
2366
2367
2368
2369 spin_lock_irq(&ndlp->lock);
2370 ndlp->nrport = NULL;
2371 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2372 spin_unlock_irq(&ndlp->lock);
2373 rport->ndlp = NULL;
2374 rport->remoteport = NULL;
2375
2376
2377
2378
2379
2380 if (prev_ndlp && prev_ndlp != ndlp) {
2381 if (!prev_ndlp->nrport)
2382 lpfc_nlp_put(prev_ndlp);
2383 }
2384 }
2385
2386
2387 rport->remoteport = remote_port;
2388 rport->lport = lport;
2389 rport->ndlp = ndlp;
2390 spin_lock_irq(&ndlp->lock);
2391 ndlp->nrport = rport;
2392 spin_unlock_irq(&ndlp->lock);
2393 lpfc_printf_vlog(vport, KERN_INFO,
2394 LOG_NVME_DISC | LOG_NODE,
2395 "6022 Bind lport x%px to remoteport x%px "
2396 "rport x%px WWNN 0x%llx, "
2397 "Rport WWPN 0x%llx DID "
2398 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2399 lport, remote_port, rport,
2400 rpinfo.node_name, rpinfo.port_name,
2401 rpinfo.port_id, rpinfo.port_role,
2402 ndlp, prev_ndlp);
2403 } else {
2404 lpfc_printf_vlog(vport, KERN_ERR,
2405 LOG_TRACE_EVENT,
2406 "6031 RemotePort Registration failed "
2407 "err: %d, DID x%06x\n",
2408 ret, ndlp->nlp_DID);
2409 }
2410
2411 return ret;
2412#else
2413 return 0;
2414#endif
2415}
2416
2417
2418
2419
2420
2421
2422
2423
2424void
2425lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2426{
2427#if (IS_ENABLED(CONFIG_NVME_FC))
2428 struct lpfc_nvme_rport *nrport;
2429 struct nvme_fc_remote_port *remoteport = NULL;
2430
2431 spin_lock_irq(&ndlp->lock);
2432 nrport = lpfc_ndlp_get_nrport(ndlp);
2433 if (nrport)
2434 remoteport = nrport->remoteport;
2435 spin_unlock_irq(&ndlp->lock);
2436
2437 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2438 "6170 Rescan NPort DID x%06x type x%x "
2439 "state x%x nrport x%px remoteport x%px\n",
2440 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2441 nrport, remoteport);
2442
2443 if (!nrport || !remoteport)
2444 goto rescan_exit;
2445
2446
2447 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2448 ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2449 nvme_fc_rescan_remoteport(remoteport);
2450
2451 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2452 "6172 NVME rescanned DID x%06x "
2453 "port_state x%x\n",
2454 ndlp->nlp_DID, remoteport->port_state);
2455 }
2456 return;
2457 rescan_exit:
2458 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2459 "6169 Skip NVME Rport Rescan, NVME remoteport "
2460 "unregistered\n");
2461#endif
2462}
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476void
2477lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2478{
2479#if (IS_ENABLED(CONFIG_NVME_FC))
2480 int ret;
2481 struct nvme_fc_local_port *localport;
2482 struct lpfc_nvme_lport *lport;
2483 struct lpfc_nvme_rport *rport;
2484 struct nvme_fc_remote_port *remoteport = NULL;
2485
2486 localport = vport->localport;
2487
2488
2489
2490
2491 if (!localport)
2492 return;
2493
2494 lport = (struct lpfc_nvme_lport *)localport->private;
2495 if (!lport)
2496 goto input_err;
2497
2498 spin_lock_irq(&ndlp->lock);
2499 rport = lpfc_ndlp_get_nrport(ndlp);
2500 if (rport)
2501 remoteport = rport->remoteport;
2502 spin_unlock_irq(&ndlp->lock);
2503 if (!remoteport)
2504 goto input_err;
2505
2506 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2507 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2508 "port_id x%06x, portstate x%x port type x%x "
2509 "refcnt %d\n",
2510 remoteport, remoteport->port_name,
2511 remoteport->port_id, remoteport->port_state,
2512 ndlp->nlp_type, kref_read(&ndlp->kref));
2513
2514
2515
2516
2517
2518 if (ndlp->nlp_type & NLP_NVME_TARGET) {
2519
2520
2521
2522 spin_lock_irq(&vport->phba->hbalock);
2523 ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2524 spin_unlock_irq(&vport->phba->hbalock);
2525
2526
2527
2528
2529
2530
2531 if (vport->load_flag & FC_UNLOADING)
2532 (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2533
2534 ret = nvme_fc_unregister_remoteport(remoteport);
2535
2536
2537
2538
2539
2540
2541 ndlp->nrport = NULL;
2542 lpfc_nlp_put(ndlp);
2543 if (ret != 0) {
2544 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2545 "6167 NVME unregister failed %d "
2546 "port_state x%x\n",
2547 ret, remoteport->port_state);
2548 }
2549 }
2550 return;
2551
2552 input_err:
2553#endif
2554 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2555 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2556 vport->localport, ndlp->rport, ndlp->nlp_DID);
2557}
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569void
2570lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2571 struct sli4_wcqe_xri_aborted *axri,
2572 struct lpfc_io_buf *lpfc_ncmd)
2573{
2574 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2575 struct nvmefc_fcp_req *nvme_cmd = NULL;
2576 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2577
2578
2579 if (ndlp)
2580 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2581
2582 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2583 "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2584 "xri released\n",
2585 lpfc_ncmd->nvmeCmd, xri,
2586 lpfc_ncmd->cur_iocbq.iotag);
2587
2588
2589
2590
2591
2592 if (lpfc_ncmd->nvmeCmd) {
2593 nvme_cmd = lpfc_ncmd->nvmeCmd;
2594 nvme_cmd->done(nvme_cmd);
2595 lpfc_ncmd->nvmeCmd = NULL;
2596 }
2597 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2598}
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610void
2611lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2612{
2613 struct lpfc_sli_ring *pring;
2614 u32 i, wait_cnt = 0;
2615
2616 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2617 return;
2618
2619
2620
2621
2622 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2623 if (!phba->sli4_hba.hdwq[i].io_wq)
2624 continue;
2625 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2626
2627 if (!pring)
2628 continue;
2629
2630
2631 while (!list_empty(&pring->txcmplq)) {
2632 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2633 wait_cnt++;
2634
2635
2636
2637
2638 if ((wait_cnt % 1000) == 0) {
2639 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2640 "6178 NVME IO not empty, "
2641 "cnt %d\n", wait_cnt);
2642 }
2643 }
2644 }
2645
2646
2647 lpfc_issue_hb_tmo(phba);
2648
2649}
2650
2651void
2652lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
2653 uint32_t stat, uint32_t param)
2654{
2655#if (IS_ENABLED(CONFIG_NVME_FC))
2656 struct lpfc_io_buf *lpfc_ncmd;
2657 struct nvmefc_fcp_req *nCmd;
2658 struct lpfc_wcqe_complete wcqe;
2659 struct lpfc_wcqe_complete *wcqep = &wcqe;
2660
2661 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2662 if (!lpfc_ncmd) {
2663 lpfc_sli_release_iocbq(phba, pwqeIn);
2664 return;
2665 }
2666
2667 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2668 CMD_ABORT_XRI_CX) {
2669 lpfc_sli_release_iocbq(phba, pwqeIn);
2670 return;
2671 }
2672
2673 spin_lock(&lpfc_ncmd->buf_lock);
2674 nCmd = lpfc_ncmd->nvmeCmd;
2675 if (!nCmd) {
2676 spin_unlock(&lpfc_ncmd->buf_lock);
2677 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2678 return;
2679 }
2680 spin_unlock(&lpfc_ncmd->buf_lock);
2681
2682 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2683 "6194 NVME Cancel xri %x\n",
2684 lpfc_ncmd->cur_iocbq.sli4_xritag);
2685
2686 wcqep->word0 = 0;
2687 bf_set(lpfc_wcqe_c_status, wcqep, stat);
2688 wcqep->parameter = param;
2689 wcqep->word3 = 0;
2690
2691
2692 if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
2693 bf_set(lpfc_wcqe_c_xb, wcqep, 1);
2694
2695 (pwqeIn->wqe_cmpl)(phba, pwqeIn, wcqep);
2696#endif
2697}
2698