1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/pci.h>
24#include <linux/slab.h>
25#include <linux/interrupt.h>
26#include <linux/delay.h>
27#include <asm/unaligned.h>
28#include <linux/crc-t10dif.h>
29#include <net/checksum.h>
30
31#include <scsi/scsi.h>
32#include <scsi/scsi_device.h>
33#include <scsi/scsi_eh.h>
34#include <scsi/scsi_host.h>
35#include <scsi/scsi_tcq.h>
36#include <scsi/scsi_transport_fc.h>
37#include <scsi/fc/fc_fs.h>
38
39#include "lpfc_version.h"
40#include "lpfc_hw4.h"
41#include "lpfc_hw.h"
42#include "lpfc_sli.h"
43#include "lpfc_sli4.h"
44#include "lpfc_nl.h"
45#include "lpfc_disc.h"
46#include "lpfc.h"
47#include "lpfc_nvme.h"
48#include "lpfc_scsi.h"
49#include "lpfc_logmsg.h"
50#include "lpfc_crtn.h"
51#include "lpfc_vport.h"
52#include "lpfc_debugfs.h"
53
54
55
56static struct lpfc_io_buf *
57lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 int idx, int expedite);
59
60static void
61lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62
63static struct nvme_fc_port_template lpfc_nvme_template;
64
65static union lpfc_wqe128 lpfc_iread_cmd_template;
66static union lpfc_wqe128 lpfc_iwrite_cmd_template;
67static union lpfc_wqe128 lpfc_icmnd_cmd_template;
68
69
70void
71lpfc_nvme_cmd_template(void)
72{
73 union lpfc_wqe128 *wqe;
74
75
76 wqe = &lpfc_iread_cmd_template;
77 memset(wqe, 0, sizeof(union lpfc_wqe128));
78
79
80
81
82
83
84
85
86
87
88
89
90 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
91 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
92 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
93 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
94
95
96
97
98
99
100 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
101 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
102 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
103 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
104 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
105 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
106
107
108 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
109 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
110 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
111
112
113
114
115
116
117 wqe = &lpfc_iwrite_cmd_template;
118 memset(wqe, 0, sizeof(union lpfc_wqe128));
119
120
121
122
123
124
125
126
127
128
129
130
131 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
132 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
133 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
134 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
135
136
137
138
139
140
141 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
142 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
143 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
144 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
145 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
146 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
147
148
149 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
150 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
151 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
152
153
154
155
156
157
158 wqe = &lpfc_icmnd_cmd_template;
159 memset(wqe, 0, sizeof(union lpfc_wqe128));
160
161
162
163
164
165
166
167
168
169
170 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
171 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
172 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
173 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
174
175
176
177
178
179
180 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
181 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
182 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
183 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
184 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
185 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
186
187
188 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
189 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
190 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
191
192
193}
194
195
196
197
198
199
200
201
202
203
204void
205lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt)
206{
207 union lpfc_wqe128 *wqe = &pwqeq->wqe;
208
209
210
211
212 memset(wqe, 0, sizeof(*wqe));
213
214 if (opt & INHIBIT_ABORT)
215 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
216
217 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
218
219 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
220
221
222 wqe->abort_cmd.wqe_com.abort_tag = xritag;
223
224
225 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag);
226
227 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
228 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
229
230 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND);
231 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1);
232 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
233}
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252static int
253lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
254 unsigned int qidx, u16 qsize,
255 void **handle)
256{
257 struct lpfc_nvme_lport *lport;
258 struct lpfc_vport *vport;
259 struct lpfc_nvme_qhandle *qhandle;
260 char *str;
261
262 if (!pnvme_lport->private)
263 return -ENOMEM;
264
265 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
266 vport = lport->vport;
267 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
268 if (qhandle == NULL)
269 return -ENOMEM;
270
271 qhandle->cpu_id = raw_smp_processor_id();
272 qhandle->qidx = qidx;
273
274
275
276
277
278 if (qidx) {
279 str = "IO ";
280 qhandle->index = ((qidx - 1) %
281 lpfc_nvme_template.max_hw_queues);
282 } else {
283 str = "ADM";
284 qhandle->index = qidx;
285 }
286
287 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
288 "6073 Binding %s HdwQueue %d (cpu %d) to "
289 "hdw_queue %d qhandle x%px\n", str,
290 qidx, qhandle->cpu_id, qhandle->index, qhandle);
291 *handle = (void *)qhandle;
292 return 0;
293}
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309static void
310lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
311 unsigned int qidx,
312 void *handle)
313{
314 struct lpfc_nvme_lport *lport;
315 struct lpfc_vport *vport;
316
317 if (!pnvme_lport->private)
318 return;
319
320 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
321 vport = lport->vport;
322
323 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
324 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
325 lport, qidx, handle);
326 kfree(handle);
327}
328
329static void
330lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
331{
332 struct lpfc_nvme_lport *lport = localport->private;
333
334 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
335 "6173 localport x%px delete complete\n",
336 lport);
337
338
339 if (lport->vport->localport)
340 complete(lport->lport_unreg_cmp);
341}
342
343
344
345
346
347
348
349
350
351
352
353
354static void
355lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
356{
357 struct lpfc_nvme_rport *rport = remoteport->private;
358 struct lpfc_vport *vport;
359 struct lpfc_nodelist *ndlp;
360
361 ndlp = rport->ndlp;
362 if (!ndlp)
363 goto rport_err;
364
365 vport = ndlp->vport;
366 if (!vport)
367 goto rport_err;
368
369
370
371
372
373 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
374 "6146 remoteport delete of remoteport x%px\n",
375 remoteport);
376 spin_lock_irq(&vport->phba->hbalock);
377
378
379
380
381 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
382 ndlp->nrport = NULL;
383 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
384 spin_unlock_irq(&vport->phba->hbalock);
385
386
387
388
389 lpfc_nlp_put(ndlp);
390 } else {
391 spin_unlock_irq(&vport->phba->hbalock);
392 }
393
394 rport_err:
395 return;
396}
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414int
415lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
416 struct lpfc_async_xchg_ctx *axchg)
417{
418#if (IS_ENABLED(CONFIG_NVME_FC))
419 struct lpfc_vport *vport;
420 struct lpfc_nvme_rport *lpfc_rport;
421 struct nvme_fc_remote_port *remoteport;
422 struct lpfc_nvme_lport *lport;
423 uint32_t *payload = axchg->payload;
424 int rc;
425
426 vport = axchg->ndlp->vport;
427 lpfc_rport = axchg->ndlp->nrport;
428 if (!lpfc_rport)
429 return -EINVAL;
430
431 remoteport = lpfc_rport->remoteport;
432 if (!vport->localport)
433 return -EINVAL;
434
435 lport = vport->localport->private;
436 if (!lport)
437 return -EINVAL;
438
439 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
440 axchg->size);
441
442 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
443 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
444 "%08x %08x %08x\n",
445 axchg->size, rc,
446 *payload, *(payload+1), *(payload+2),
447 *(payload+3), *(payload+4), *(payload+5));
448
449 if (!rc)
450 return 0;
451#endif
452 return 1;
453}
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468void
469__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
470 struct lpfc_iocbq *cmdwqe,
471 struct lpfc_wcqe_complete *wcqe)
472{
473 struct nvmefc_ls_req *pnvme_lsreq;
474 struct lpfc_dmabuf *buf_ptr;
475 struct lpfc_nodelist *ndlp;
476 uint32_t status;
477
478 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
479 ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
480 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
481
482 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
483 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x "
484 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
485 "ndlp:x%px\n",
486 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
487 cmdwqe->sli4_xritag, status,
488 (wcqe->parameter & 0xffff),
489 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
490
491 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
492 cmdwqe->sli4_xritag, status, wcqe->parameter);
493
494 if (cmdwqe->context3) {
495 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
496 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
497 kfree(buf_ptr);
498 cmdwqe->context3 = NULL;
499 }
500 if (pnvme_lsreq->done)
501 pnvme_lsreq->done(pnvme_lsreq, status);
502 else
503 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
504 "6046 NVMEx cmpl without done call back? "
505 "Data %px DID %x Xri: %x status %x\n",
506 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
507 cmdwqe->sli4_xritag, status);
508 if (ndlp) {
509 lpfc_nlp_put(ndlp);
510 cmdwqe->context1 = NULL;
511 }
512 lpfc_sli_release_iocbq(phba, cmdwqe);
513}
514
515static void
516lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
517 struct lpfc_wcqe_complete *wcqe)
518{
519 struct lpfc_vport *vport = cmdwqe->vport;
520 struct lpfc_nvme_lport *lport;
521 uint32_t status;
522
523 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
524
525 if (vport->localport) {
526 lport = (struct lpfc_nvme_lport *)vport->localport->private;
527 if (lport) {
528 atomic_inc(&lport->fc4NvmeLsCmpls);
529 if (status) {
530 if (bf_get(lpfc_wcqe_c_xb, wcqe))
531 atomic_inc(&lport->cmpl_ls_xb);
532 atomic_inc(&lport->cmpl_ls_err);
533 }
534 }
535 }
536
537 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
538}
539
540static int
541lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
542 struct lpfc_dmabuf *inp,
543 struct nvmefc_ls_req *pnvme_lsreq,
544 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
545 struct lpfc_wcqe_complete *),
546 struct lpfc_nodelist *ndlp, uint32_t num_entry,
547 uint32_t tmo, uint8_t retry)
548{
549 struct lpfc_hba *phba = vport->phba;
550 union lpfc_wqe128 *wqe;
551 struct lpfc_iocbq *genwqe;
552 struct ulp_bde64 *bpl;
553 struct ulp_bde64 bde;
554 int i, rc, xmit_len, first_len;
555
556
557 genwqe = lpfc_sli_get_iocbq(phba);
558 if (genwqe == NULL)
559 return 1;
560
561 wqe = &genwqe->wqe;
562
563 memset(wqe, 0, sizeof(union lpfc_wqe));
564
565 genwqe->context3 = (uint8_t *)bmp;
566 genwqe->iocb_flag |= LPFC_IO_NVME_LS;
567
568
569 genwqe->context1 = lpfc_nlp_get(ndlp);
570 genwqe->context2 = (uint8_t *)pnvme_lsreq;
571
572
573 if (!tmo)
574
575 tmo = (3 * phba->fc_ratov);
576
577
578 xmit_len = 0;
579 first_len = 0;
580 bpl = (struct ulp_bde64 *)bmp->virt;
581 for (i = 0; i < num_entry; i++) {
582 bde.tus.w = bpl[i].tus.w;
583 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
584 break;
585 xmit_len += bde.tus.f.bdeSize;
586 if (i == 0)
587 first_len = xmit_len;
588 }
589
590 genwqe->rsvd2 = num_entry;
591 genwqe->hba_wqidx = 0;
592
593
594 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
595 wqe->generic.bde.tus.f.bdeSize = first_len;
596 wqe->generic.bde.addrLow = bpl[0].addrLow;
597 wqe->generic.bde.addrHigh = bpl[0].addrHigh;
598
599
600 wqe->gen_req.request_payload_len = first_len;
601
602
603
604
605 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
606 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
607 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
608 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
609 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
610
611
612 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
613 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
614 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
615
616
617 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
618 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
619 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
620 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
621
622
623 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
624
625
626 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
627
628
629 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
630 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
631 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
632 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
633 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
634
635
636 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
637 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
638
639
640
641 genwqe->wqe_cmpl = cmpl;
642 genwqe->iocb_cmpl = NULL;
643 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
644 genwqe->vport = vport;
645 genwqe->retry = retry;
646
647 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
648 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
649
650 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
651 if (rc) {
652 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
653 "6045 Issue GEN REQ WQE to NPORT x%x "
654 "Data: x%x x%x rc x%x\n",
655 ndlp->nlp_DID, genwqe->iotag,
656 vport->port_state, rc);
657 lpfc_sli_release_iocbq(phba, genwqe);
658 return 1;
659 }
660
661 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
662 "6050 Issue GEN REQ WQE to NPORT x%x "
663 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
664 "bmp:x%px xmit:%d 1st:%d\n",
665 ndlp->nlp_DID, genwqe->sli4_xritag,
666 vport->port_state,
667 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
668 return 0;
669}
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686int
687__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
688 struct nvmefc_ls_req *pnvme_lsreq,
689 void (*gen_req_cmp)(struct lpfc_hba *phba,
690 struct lpfc_iocbq *cmdwqe,
691 struct lpfc_wcqe_complete *wcqe))
692{
693 struct lpfc_dmabuf *bmp;
694 struct ulp_bde64 *bpl;
695 int ret;
696 uint16_t ntype, nstate;
697
698 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
699 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
700 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
701 "LS Req\n",
702 ndlp);
703 return -ENODEV;
704 }
705
706 ntype = ndlp->nlp_type;
707 nstate = ndlp->nlp_state;
708 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
709 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
710 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
711 "6088 NVMEx LS REQ: Fail DID x%06x not "
712 "ready for IO. Type x%x, State x%x\n",
713 ndlp->nlp_DID, ntype, nstate);
714 return -ENODEV;
715 }
716
717
718
719
720
721
722
723
724
725
726
727
728
729 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
730 if (!bmp) {
731 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
732 "6044 NVMEx LS REQ: Could not alloc LS buf "
733 "for DID %x\n",
734 ndlp->nlp_DID);
735 return -ENOMEM;
736 }
737
738 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
739 if (!bmp->virt) {
740 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
741 "6042 NVMEx LS REQ: Could not alloc mbuf "
742 "for DID %x\n",
743 ndlp->nlp_DID);
744 kfree(bmp);
745 return -ENOMEM;
746 }
747
748 INIT_LIST_HEAD(&bmp->list);
749
750 bpl = (struct ulp_bde64 *)bmp->virt;
751 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
752 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
753 bpl->tus.f.bdeFlags = 0;
754 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
755 bpl->tus.w = le32_to_cpu(bpl->tus.w);
756 bpl++;
757
758 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
759 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
760 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
761 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
762 bpl->tus.w = le32_to_cpu(bpl->tus.w);
763
764 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
765 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
766 "rqstlen:%d rsplen:%d %pad %pad\n",
767 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
768 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
769 &pnvme_lsreq->rspdma);
770
771 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
772 pnvme_lsreq, gen_req_cmp, ndlp, 2,
773 LPFC_NVME_LS_TIMEOUT, 0);
774 if (ret != WQE_SUCCESS) {
775 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
776 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
777 "lsreq x%px Status %x DID %x\n",
778 pnvme_lsreq, ret, ndlp->nlp_DID);
779 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
780 kfree(bmp);
781 return -EIO;
782 }
783
784 return 0;
785}
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800static int
801lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
802 struct nvme_fc_remote_port *pnvme_rport,
803 struct nvmefc_ls_req *pnvme_lsreq)
804{
805 struct lpfc_nvme_lport *lport;
806 struct lpfc_nvme_rport *rport;
807 struct lpfc_vport *vport;
808 int ret;
809
810 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
811 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
812 if (unlikely(!lport) || unlikely(!rport))
813 return -EINVAL;
814
815 vport = lport->vport;
816 if (vport->load_flag & FC_UNLOADING)
817 return -ENODEV;
818
819 atomic_inc(&lport->fc4NvmeLsRequests);
820
821 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
822 lpfc_nvme_ls_req_cmp);
823 if (ret)
824 atomic_inc(&lport->xmt_ls_err);
825
826 return ret;
827}
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843int
844__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
845 struct nvmefc_ls_req *pnvme_lsreq)
846{
847 struct lpfc_hba *phba = vport->phba;
848 struct lpfc_sli_ring *pring;
849 struct lpfc_iocbq *wqe, *next_wqe;
850 bool foundit = false;
851
852 if (!ndlp) {
853 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
854 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
855 "x%06x, Failing LS Req\n",
856 ndlp, ndlp ? ndlp->nlp_DID : 0);
857 return -EINVAL;
858 }
859
860 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
861 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
862 "x%p rqstlen:%d rsplen:%d %pad %pad\n",
863 pnvme_lsreq, pnvme_lsreq->rqstlen,
864 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
865 &pnvme_lsreq->rspdma);
866
867
868
869
870
871 pring = phba->sli4_hba.nvmels_wq->pring;
872 spin_lock_irq(&phba->hbalock);
873 spin_lock(&pring->ring_lock);
874 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
875 if (wqe->context2 == pnvme_lsreq) {
876 wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
877 foundit = true;
878 break;
879 }
880 }
881 spin_unlock(&pring->ring_lock);
882
883 if (foundit)
884 lpfc_sli_issue_abort_iotag(phba, pring, wqe);
885 spin_unlock_irq(&phba->hbalock);
886
887 if (foundit)
888 return 0;
889
890 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
891 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n",
892 pnvme_lsreq);
893 return -EINVAL;
894}
895
896static int
897lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
898 struct nvme_fc_remote_port *remoteport,
899 struct nvmefc_ls_rsp *ls_rsp)
900{
901 struct lpfc_async_xchg_ctx *axchg =
902 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
903 struct lpfc_nvme_lport *lport;
904 int rc;
905
906 if (axchg->phba->pport->load_flag & FC_UNLOADING)
907 return -ENODEV;
908
909 lport = (struct lpfc_nvme_lport *)localport->private;
910
911 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
912
913 if (rc) {
914
915
916
917
918
919 if (rc != -EALREADY)
920 atomic_inc(&lport->xmt_ls_abort);
921 return rc;
922 }
923
924 return 0;
925}
926
927
928
929
930
931
932
933
934
935
936static void
937lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
938 struct nvme_fc_remote_port *pnvme_rport,
939 struct nvmefc_ls_req *pnvme_lsreq)
940{
941 struct lpfc_nvme_lport *lport;
942 struct lpfc_vport *vport;
943 struct lpfc_hba *phba;
944 struct lpfc_nodelist *ndlp;
945 int ret;
946
947 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
948 if (unlikely(!lport))
949 return;
950 vport = lport->vport;
951 phba = vport->phba;
952
953 if (vport->load_flag & FC_UNLOADING)
954 return;
955
956 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
957
958 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
959 if (!ret)
960 atomic_inc(&lport->xmt_ls_abort);
961}
962
963
964static inline void
965lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
966 struct lpfc_io_buf *lpfc_ncmd,
967 struct nvmefc_fcp_req *nCmd)
968{
969 struct lpfc_hba *phba = vport->phba;
970 struct sli4_sge *sgl;
971 union lpfc_wqe128 *wqe;
972 uint32_t *wptr, *dptr;
973
974
975
976
977
978
979
980 wqe = &lpfc_ncmd->cur_iocbq.wqe;
981
982
983
984
985
986
987
988
989 sgl = lpfc_ncmd->dma_sgl;
990 sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
991 if (phba->cfg_nvme_embed_cmd) {
992 sgl->addr_hi = 0;
993 sgl->addr_lo = 0;
994
995
996 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
997 wqe->generic.bde.tus.f.bdeSize = 56;
998 wqe->generic.bde.addrHigh = 0;
999 wqe->generic.bde.addrLow = 64;
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011 wptr = &wqe->words[16];
1012 dptr = (uint32_t *)nCmd->cmdaddr;
1013 dptr++;
1014
1015 *wptr++ = *dptr++;
1016 *wptr++ = *dptr++;
1017 *wptr++ = *dptr++;
1018 *wptr++ = *dptr++;
1019 dptr++;
1020 *wptr++ = *dptr++;
1021 *wptr++ = *dptr++;
1022 dptr += 8;
1023 *wptr++ = *dptr++;
1024 *wptr++ = *dptr++;
1025 *wptr++ = *dptr++;
1026 *wptr++ = *dptr++;
1027 *wptr++ = *dptr++;
1028 *wptr++ = *dptr++;
1029 *wptr++ = *dptr++;
1030 *wptr = *dptr;
1031 } else {
1032 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
1033 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
1034
1035
1036 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1037 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
1038 wqe->generic.bde.addrHigh = sgl->addr_hi;
1039 wqe->generic.bde.addrLow = sgl->addr_lo;
1040
1041
1042 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
1043 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
1044 }
1045
1046 sgl++;
1047
1048
1049 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
1050 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
1051 sgl->word2 = le32_to_cpu(sgl->word2);
1052 if (nCmd->sg_cnt)
1053 bf_set(lpfc_sli4_sge_last, sgl, 0);
1054 else
1055 bf_set(lpfc_sli4_sge_last, sgl, 1);
1056 sgl->word2 = cpu_to_le32(sgl->word2);
1057 sgl->sge_len = cpu_to_le32(nCmd->rsplen);
1058}
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072static void
1073lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
1074 struct lpfc_wcqe_complete *wcqe)
1075{
1076 struct lpfc_io_buf *lpfc_ncmd =
1077 (struct lpfc_io_buf *)pwqeIn->context1;
1078 struct lpfc_vport *vport = pwqeIn->vport;
1079 struct nvmefc_fcp_req *nCmd;
1080 struct nvme_fc_ersp_iu *ep;
1081 struct nvme_fc_cmd_iu *cp;
1082 struct lpfc_nodelist *ndlp;
1083 struct lpfc_nvme_fcpreq_priv *freqpriv;
1084 struct lpfc_nvme_lport *lport;
1085 uint32_t code, status, idx;
1086 uint16_t cid, sqhd, data;
1087 uint32_t *ptr;
1088#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1089 int cpu;
1090#endif
1091
1092
1093 if (!lpfc_ncmd) {
1094 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1095 "6071 Null lpfc_ncmd pointer. No "
1096 "release, skip completion\n");
1097 return;
1098 }
1099
1100
1101 spin_lock(&lpfc_ncmd->buf_lock);
1102
1103 if (!lpfc_ncmd->nvmeCmd) {
1104 spin_unlock(&lpfc_ncmd->buf_lock);
1105 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1106 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
1107 "nvmeCmd x%px\n",
1108 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
1109
1110
1111 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1112 return;
1113 }
1114 nCmd = lpfc_ncmd->nvmeCmd;
1115 status = bf_get(lpfc_wcqe_c_status, wcqe);
1116
1117 idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1118 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
1119
1120 if (unlikely(status && vport->localport)) {
1121 lport = (struct lpfc_nvme_lport *)vport->localport->private;
1122 if (lport) {
1123 if (bf_get(lpfc_wcqe_c_xb, wcqe))
1124 atomic_inc(&lport->cmpl_fcp_xb);
1125 atomic_inc(&lport->cmpl_fcp_err);
1126 }
1127 }
1128
1129 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
1130 lpfc_ncmd->cur_iocbq.sli4_xritag,
1131 status, wcqe->parameter);
1132
1133
1134
1135
1136 ndlp = lpfc_ncmd->ndlp;
1137 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1138 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1139 "6062 Ignoring NVME cmpl. No ndlp\n");
1140 goto out_err;
1141 }
1142
1143 code = bf_get(lpfc_wcqe_c_code, wcqe);
1144 if (code == CQE_CODE_NVME_ERSP) {
1145
1146 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1147
1148
1149
1150
1151
1152 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1153 cid = cp->sqe.common.command_id;
1154
1155
1156
1157
1158
1159
1160
1161 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1162
1163
1164 ep->iu_len = cpu_to_be16(8);
1165 ep->rsn = wcqe->parameter;
1166 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1167 ep->rsvd12 = 0;
1168 ptr = (uint32_t *)&ep->cqe.result.u64;
1169 *ptr++ = wcqe->total_data_placed;
1170 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1171 *ptr = (uint32_t)data;
1172 ep->cqe.sq_head = sqhd;
1173 ep->cqe.sq_id = nCmd->sqid;
1174 ep->cqe.command_id = cid;
1175 ep->cqe.status = 0;
1176
1177 lpfc_ncmd->status = IOSTAT_SUCCESS;
1178 lpfc_ncmd->result = 0;
1179 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1180 nCmd->transferred_length = nCmd->payload_length;
1181 } else {
1182 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1183 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196 switch (lpfc_ncmd->status) {
1197 case IOSTAT_SUCCESS:
1198 nCmd->transferred_length = wcqe->total_data_placed;
1199 nCmd->rcv_rsplen = 0;
1200 nCmd->status = 0;
1201 break;
1202 case IOSTAT_FCP_RSP_ERROR:
1203 nCmd->transferred_length = wcqe->total_data_placed;
1204 nCmd->rcv_rsplen = wcqe->parameter;
1205 nCmd->status = 0;
1206
1207 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
1208 break;
1209 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1210 "6081 NVME Completion Protocol Error: "
1211 "xri %x status x%x result x%x "
1212 "placed x%x\n",
1213 lpfc_ncmd->cur_iocbq.sli4_xritag,
1214 lpfc_ncmd->status, lpfc_ncmd->result,
1215 wcqe->total_data_placed);
1216 break;
1217 case IOSTAT_LOCAL_REJECT:
1218
1219 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1220 lpfc_printf_vlog(vport, KERN_INFO,
1221 LOG_NVME_IOERR,
1222 "6032 Delay Aborted cmd x%px "
1223 "nvme cmd x%px, xri x%x, "
1224 "xb %d\n",
1225 lpfc_ncmd, nCmd,
1226 lpfc_ncmd->cur_iocbq.sli4_xritag,
1227 bf_get(lpfc_wcqe_c_xb, wcqe));
1228 fallthrough;
1229 default:
1230out_err:
1231 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1232 "6072 NVME Completion Error: xri %x "
1233 "status x%x result x%x [x%x] "
1234 "placed x%x\n",
1235 lpfc_ncmd->cur_iocbq.sli4_xritag,
1236 lpfc_ncmd->status, lpfc_ncmd->result,
1237 wcqe->parameter,
1238 wcqe->total_data_placed);
1239 nCmd->transferred_length = 0;
1240 nCmd->rcv_rsplen = 0;
1241 nCmd->status = NVME_SC_INTERNAL;
1242 }
1243 }
1244
1245
1246 if (bf_get(lpfc_wcqe_c_xb, wcqe))
1247 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1248 else
1249 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1250
1251
1252
1253
1254
1255#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1256 if (lpfc_ncmd->ts_cmd_start) {
1257 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1258 lpfc_ncmd->ts_data_io = ktime_get_ns();
1259 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1260 lpfc_io_ktime(phba, lpfc_ncmd);
1261 }
1262 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1263 cpu = raw_smp_processor_id();
1264 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1265 if (lpfc_ncmd->cpu != cpu)
1266 lpfc_printf_vlog(vport,
1267 KERN_INFO, LOG_NVME_IOERR,
1268 "6701 CPU Check cmpl: "
1269 "cpu %d expect %d\n",
1270 cpu, lpfc_ncmd->cpu);
1271 }
1272#endif
1273
1274
1275
1276
1277
1278 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1279 freqpriv = nCmd->private;
1280 freqpriv->nvme_buf = NULL;
1281 lpfc_ncmd->nvmeCmd = NULL;
1282 spin_unlock(&lpfc_ncmd->buf_lock);
1283 nCmd->done(nCmd);
1284 } else
1285 spin_unlock(&lpfc_ncmd->buf_lock);
1286
1287
1288 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1289}
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307static int
1308lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1309 struct lpfc_io_buf *lpfc_ncmd,
1310 struct lpfc_nodelist *pnode,
1311 struct lpfc_fc4_ctrl_stat *cstat)
1312{
1313 struct lpfc_hba *phba = vport->phba;
1314 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1315 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1316 union lpfc_wqe128 *wqe = &pwqeq->wqe;
1317 uint32_t req_len;
1318
1319 if (!NLP_CHK_NODE_ACT(pnode))
1320 return -EINVAL;
1321
1322
1323
1324
1325
1326 if (nCmd->sg_cnt) {
1327 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1328
1329 memcpy(&wqe->words[7],
1330 &lpfc_iwrite_cmd_template.words[7],
1331 sizeof(uint32_t) * 5);
1332
1333
1334 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1335
1336
1337 if ((phba->cfg_nvme_enable_fb) &&
1338 (pnode->nlp_flag & NLP_FIRSTBURST)) {
1339 req_len = lpfc_ncmd->nvmeCmd->payload_length;
1340 if (req_len < pnode->nvme_fb_size)
1341 wqe->fcp_iwrite.initial_xfer_len =
1342 req_len;
1343 else
1344 wqe->fcp_iwrite.initial_xfer_len =
1345 pnode->nvme_fb_size;
1346 } else {
1347 wqe->fcp_iwrite.initial_xfer_len = 0;
1348 }
1349 cstat->output_requests++;
1350 } else {
1351
1352 memcpy(&wqe->words[7],
1353 &lpfc_iread_cmd_template.words[7],
1354 sizeof(uint32_t) * 5);
1355
1356
1357 wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1358
1359
1360 wqe->fcp_iread.rsrvd5 = 0;
1361
1362 cstat->input_requests++;
1363 }
1364 } else {
1365
1366 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1367 sizeof(uint32_t) * 8);
1368 cstat->control_requests++;
1369 }
1370
1371 if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1372 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1373
1374
1375
1376
1377
1378
1379 bf_set(payload_offset_len, &wqe->fcp_icmd,
1380 (nCmd->rsplen + nCmd->cmdlen));
1381
1382
1383 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1384 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1385 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1386
1387
1388 wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1389
1390
1391 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1392
1393
1394
1395 pwqeq->vport = vport;
1396 return 0;
1397}
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413static int
1414lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1415 struct lpfc_io_buf *lpfc_ncmd)
1416{
1417 struct lpfc_hba *phba = vport->phba;
1418 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1419 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1420 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1421 struct sli4_hybrid_sgl *sgl_xtra = NULL;
1422 struct scatterlist *data_sg;
1423 struct sli4_sge *first_data_sgl;
1424 struct ulp_bde64 *bde;
1425 dma_addr_t physaddr = 0;
1426 uint32_t num_bde = 0;
1427 uint32_t dma_len = 0;
1428 uint32_t dma_offset = 0;
1429 int nseg, i, j;
1430 bool lsp_just_set = false;
1431
1432
1433 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1434
1435
1436
1437
1438
1439 if (nCmd->sg_cnt) {
1440
1441
1442
1443
1444 sgl += 2;
1445
1446 first_data_sgl = sgl;
1447 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1448 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1449 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1450 "6058 Too many sg segments from "
1451 "NVME Transport. Max %d, "
1452 "nvmeIO sg_cnt %d\n",
1453 phba->cfg_nvme_seg_cnt + 1,
1454 lpfc_ncmd->seg_cnt);
1455 lpfc_ncmd->seg_cnt = 0;
1456 return 1;
1457 }
1458
1459
1460
1461
1462
1463
1464
1465 nseg = nCmd->sg_cnt;
1466 data_sg = nCmd->first_sgl;
1467
1468
1469 j = 2;
1470 for (i = 0; i < nseg; i++) {
1471 if (data_sg == NULL) {
1472 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1473 "6059 dptr err %d, nseg %d\n",
1474 i, nseg);
1475 lpfc_ncmd->seg_cnt = 0;
1476 return 1;
1477 }
1478
1479 sgl->word2 = 0;
1480 if ((num_bde + 1) == nseg) {
1481 bf_set(lpfc_sli4_sge_last, sgl, 1);
1482 bf_set(lpfc_sli4_sge_type, sgl,
1483 LPFC_SGE_TYPE_DATA);
1484 } else {
1485 bf_set(lpfc_sli4_sge_last, sgl, 0);
1486
1487
1488 if (!lsp_just_set &&
1489 !((j + 1) % phba->border_sge_num) &&
1490 ((nseg - 1) != i)) {
1491
1492 bf_set(lpfc_sli4_sge_type, sgl,
1493 LPFC_SGE_TYPE_LSP);
1494
1495 sgl_xtra = lpfc_get_sgl_per_hdwq(
1496 phba, lpfc_ncmd);
1497
1498 if (unlikely(!sgl_xtra)) {
1499 lpfc_ncmd->seg_cnt = 0;
1500 return 1;
1501 }
1502 sgl->addr_lo = cpu_to_le32(putPaddrLow(
1503 sgl_xtra->dma_phys_sgl));
1504 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1505 sgl_xtra->dma_phys_sgl));
1506
1507 } else {
1508 bf_set(lpfc_sli4_sge_type, sgl,
1509 LPFC_SGE_TYPE_DATA);
1510 }
1511 }
1512
1513 if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1514 LPFC_SGE_TYPE_LSP)) {
1515 if ((nseg - 1) == i)
1516 bf_set(lpfc_sli4_sge_last, sgl, 1);
1517
1518 physaddr = data_sg->dma_address;
1519 dma_len = data_sg->length;
1520 sgl->addr_lo = cpu_to_le32(
1521 putPaddrLow(physaddr));
1522 sgl->addr_hi = cpu_to_le32(
1523 putPaddrHigh(physaddr));
1524
1525 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1526 sgl->word2 = cpu_to_le32(sgl->word2);
1527 sgl->sge_len = cpu_to_le32(dma_len);
1528
1529 dma_offset += dma_len;
1530 data_sg = sg_next(data_sg);
1531
1532 sgl++;
1533
1534 lsp_just_set = false;
1535 } else {
1536 sgl->word2 = cpu_to_le32(sgl->word2);
1537
1538 sgl->sge_len = cpu_to_le32(
1539 phba->cfg_sg_dma_buf_size);
1540
1541 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1542 i = i - 1;
1543
1544 lsp_just_set = true;
1545 }
1546
1547 j++;
1548 }
1549 if (phba->cfg_enable_pbde) {
1550
1551
1552 bde = (struct ulp_bde64 *)
1553 &wqe->words[13];
1554 bde->addrLow = first_data_sgl->addr_lo;
1555 bde->addrHigh = first_data_sgl->addr_hi;
1556 bde->tus.f.bdeSize =
1557 le32_to_cpu(first_data_sgl->sge_len);
1558 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1559 bde->tus.w = cpu_to_le32(bde->tus.w);
1560
1561 } else {
1562 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1563 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1564 }
1565
1566 } else {
1567 lpfc_ncmd->seg_cnt = 0;
1568
1569
1570
1571
1572 if (nCmd->payload_length != 0) {
1573 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1574 "6063 NVME DMA Prep Err: sg_cnt %d "
1575 "payload_length x%x\n",
1576 nCmd->sg_cnt, nCmd->payload_length);
1577 return 1;
1578 }
1579 }
1580 return 0;
1581}
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600static int
1601lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1602 struct nvme_fc_remote_port *pnvme_rport,
1603 void *hw_queue_handle,
1604 struct nvmefc_fcp_req *pnvme_fcreq)
1605{
1606 int ret = 0;
1607 int expedite = 0;
1608 int idx, cpu;
1609 struct lpfc_nvme_lport *lport;
1610 struct lpfc_fc4_ctrl_stat *cstat;
1611 struct lpfc_vport *vport;
1612 struct lpfc_hba *phba;
1613 struct lpfc_nodelist *ndlp;
1614 struct lpfc_io_buf *lpfc_ncmd;
1615 struct lpfc_nvme_rport *rport;
1616 struct lpfc_nvme_qhandle *lpfc_queue_info;
1617 struct lpfc_nvme_fcpreq_priv *freqpriv;
1618 struct nvme_common_command *sqe;
1619#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1620 uint64_t start = 0;
1621#endif
1622
1623
1624
1625
1626 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1627 if (unlikely(!lport)) {
1628 ret = -EINVAL;
1629 goto out_fail;
1630 }
1631
1632 vport = lport->vport;
1633
1634 if (unlikely(!hw_queue_handle)) {
1635 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1636 "6117 Fail IO, NULL hw_queue_handle\n");
1637 atomic_inc(&lport->xmt_fcp_err);
1638 ret = -EBUSY;
1639 goto out_fail;
1640 }
1641
1642 phba = vport->phba;
1643
1644 if (unlikely(vport->load_flag & FC_UNLOADING)) {
1645 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1646 "6124 Fail IO, Driver unload\n");
1647 atomic_inc(&lport->xmt_fcp_err);
1648 ret = -ENODEV;
1649 goto out_fail;
1650 }
1651
1652 freqpriv = pnvme_fcreq->private;
1653 if (unlikely(!freqpriv)) {
1654 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1655 "6158 Fail IO, NULL request data\n");
1656 atomic_inc(&lport->xmt_fcp_err);
1657 ret = -EINVAL;
1658 goto out_fail;
1659 }
1660
1661#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1662 if (phba->ktime_on)
1663 start = ktime_get_ns();
1664#endif
1665 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1666 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1667
1668
1669
1670
1671
1672 ndlp = rport->ndlp;
1673 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1674 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1675 "6053 Busy IO, ndlp not ready: rport x%px "
1676 "ndlp x%px, DID x%06x\n",
1677 rport, ndlp, pnvme_rport->port_id);
1678 atomic_inc(&lport->xmt_fcp_err);
1679 ret = -EBUSY;
1680 goto out_fail;
1681 }
1682
1683
1684 if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1685 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1686 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1687 "6036 Fail IO, DID x%06x not ready for "
1688 "IO. State x%x, Type x%x Flg x%x\n",
1689 pnvme_rport->port_id,
1690 ndlp->nlp_state, ndlp->nlp_type,
1691 ndlp->upcall_flags);
1692 atomic_inc(&lport->xmt_fcp_bad_ndlp);
1693 ret = -EBUSY;
1694 goto out_fail;
1695
1696 }
1697
1698
1699
1700
1701
1702 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1703 sqe = &((struct nvme_fc_cmd_iu *)
1704 pnvme_fcreq->cmdaddr)->sqe.common;
1705 if (sqe->opcode == nvme_admin_keep_alive)
1706 expedite = 1;
1707 }
1708
1709
1710
1711
1712 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1713 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1714 !expedite) {
1715 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1716 "6174 Fail IO, ndlp qdepth exceeded: "
1717 "idx %d DID %x pend %d qdepth %d\n",
1718 lpfc_queue_info->index, ndlp->nlp_DID,
1719 atomic_read(&ndlp->cmd_pending),
1720 ndlp->cmd_qdepth);
1721 atomic_inc(&lport->xmt_fcp_qdepth);
1722 ret = -EBUSY;
1723 goto out_fail;
1724 }
1725 }
1726
1727
1728 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1729 idx = lpfc_queue_info->index;
1730 } else {
1731 cpu = raw_smp_processor_id();
1732 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1733 }
1734
1735 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1736 if (lpfc_ncmd == NULL) {
1737 atomic_inc(&lport->xmt_fcp_noxri);
1738 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1739 "6065 Fail IO, driver buffer pool is empty: "
1740 "idx %d DID %x\n",
1741 lpfc_queue_info->index, ndlp->nlp_DID);
1742 ret = -EBUSY;
1743 goto out_fail;
1744 }
1745#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1746 if (start) {
1747 lpfc_ncmd->ts_cmd_start = start;
1748 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1749 } else {
1750 lpfc_ncmd->ts_cmd_start = 0;
1751 }
1752#endif
1753
1754
1755
1756
1757
1758
1759
1760 freqpriv->nvme_buf = lpfc_ncmd;
1761 lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1762 lpfc_ncmd->ndlp = ndlp;
1763 lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773 lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1774 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1775
1776 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1777 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1778 if (ret) {
1779 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1780 "6175 Fail IO, Prep DMA: "
1781 "idx %d DID %x\n",
1782 lpfc_queue_info->index, ndlp->nlp_DID);
1783 atomic_inc(&lport->xmt_fcp_err);
1784 ret = -ENOMEM;
1785 goto out_free_nvme_buf;
1786 }
1787
1788 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1789 lpfc_ncmd->cur_iocbq.sli4_xritag,
1790 lpfc_queue_info->index, ndlp->nlp_DID);
1791
1792 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1793 if (ret) {
1794 atomic_inc(&lport->xmt_fcp_wqerr);
1795 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1796 "6113 Fail IO, Could not issue WQE err %x "
1797 "sid: x%x did: x%x oxid: x%x\n",
1798 ret, vport->fc_myDID, ndlp->nlp_DID,
1799 lpfc_ncmd->cur_iocbq.sli4_xritag);
1800 goto out_free_nvme_buf;
1801 }
1802
1803 if (phba->cfg_xri_rebalancing)
1804 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1805
1806#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1807 if (lpfc_ncmd->ts_cmd_start)
1808 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1809
1810 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1811 cpu = raw_smp_processor_id();
1812 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1813 lpfc_ncmd->cpu = cpu;
1814 if (idx != cpu)
1815 lpfc_printf_vlog(vport,
1816 KERN_INFO, LOG_NVME_IOERR,
1817 "6702 CPU Check cmd: "
1818 "cpu %d wq %d\n",
1819 lpfc_ncmd->cpu,
1820 lpfc_queue_info->index);
1821 }
1822#endif
1823 return 0;
1824
1825 out_free_nvme_buf:
1826 if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1827 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1828 cstat->output_requests--;
1829 else
1830 cstat->input_requests--;
1831 } else
1832 cstat->control_requests--;
1833 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1834 out_fail:
1835 return ret;
1836}
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849void
1850lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1851 struct lpfc_wcqe_complete *abts_cmpl)
1852{
1853 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1854 "6145 ABORT_XRI_CN completing on rpi x%x "
1855 "original iotag x%x, abort cmd iotag x%x "
1856 "req_tag x%x, status x%x, hwstatus x%x\n",
1857 cmdiocb->iocb.un.acxri.abortContextTag,
1858 cmdiocb->iocb.un.acxri.abortIoTag,
1859 cmdiocb->iotag,
1860 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1861 bf_get(lpfc_wcqe_c_status, abts_cmpl),
1862 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1863 lpfc_sli_release_iocbq(phba, cmdiocb);
1864}
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883static void
1884lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1885 struct nvme_fc_remote_port *pnvme_rport,
1886 void *hw_queue_handle,
1887 struct nvmefc_fcp_req *pnvme_fcreq)
1888{
1889 struct lpfc_nvme_lport *lport;
1890 struct lpfc_vport *vport;
1891 struct lpfc_hba *phba;
1892 struct lpfc_io_buf *lpfc_nbuf;
1893 struct lpfc_iocbq *abts_buf;
1894 struct lpfc_iocbq *nvmereq_wqe;
1895 struct lpfc_nvme_fcpreq_priv *freqpriv;
1896 unsigned long flags;
1897 int ret_val;
1898
1899
1900
1901
1902 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1903 if (unlikely(!lport))
1904 return;
1905
1906 vport = lport->vport;
1907
1908 if (unlikely(!hw_queue_handle)) {
1909 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1910 "6129 Fail Abort, HW Queue Handle NULL.\n");
1911 return;
1912 }
1913
1914 phba = vport->phba;
1915 freqpriv = pnvme_fcreq->private;
1916
1917 if (unlikely(!freqpriv))
1918 return;
1919 if (vport->load_flag & FC_UNLOADING)
1920 return;
1921
1922
1923 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1924 "6002 Abort Request to rport DID x%06x "
1925 "for nvme_fc_req x%px\n",
1926 pnvme_rport->port_id,
1927 pnvme_fcreq);
1928
1929
1930
1931
1932 spin_lock_irqsave(&phba->hbalock, flags);
1933
1934 if (phba->hba_flag & HBA_IOQ_FLUSH) {
1935 spin_unlock_irqrestore(&phba->hbalock, flags);
1936 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1937 "6139 Driver in reset cleanup - flushing "
1938 "NVME Req now. hba_flag x%x\n",
1939 phba->hba_flag);
1940 return;
1941 }
1942
1943 lpfc_nbuf = freqpriv->nvme_buf;
1944 if (!lpfc_nbuf) {
1945 spin_unlock_irqrestore(&phba->hbalock, flags);
1946 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1947 "6140 NVME IO req has no matching lpfc nvme "
1948 "io buffer. Skipping abort req.\n");
1949 return;
1950 } else if (!lpfc_nbuf->nvmeCmd) {
1951 spin_unlock_irqrestore(&phba->hbalock, flags);
1952 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1953 "6141 lpfc NVME IO req has no nvme_fcreq "
1954 "io buffer. Skipping abort req.\n");
1955 return;
1956 }
1957 nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1958
1959
1960 spin_lock(&lpfc_nbuf->buf_lock);
1961
1962
1963
1964
1965
1966
1967
1968
1969 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1970 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1971 "6143 NVME req mismatch: "
1972 "lpfc_nbuf x%px nvmeCmd x%px, "
1973 "pnvme_fcreq x%px. Skipping Abort xri x%x\n",
1974 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1975 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1976 goto out_unlock;
1977 }
1978
1979
1980 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1981 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1982 "6142 NVME IO req x%px not queued - skipping "
1983 "abort req xri x%x\n",
1984 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1985 goto out_unlock;
1986 }
1987
1988 atomic_inc(&lport->xmt_fcp_abort);
1989 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1990 nvmereq_wqe->sli4_xritag,
1991 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1992
1993
1994 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
1995 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1996 "6144 Outstanding NVME I/O Abort Request "
1997 "still pending on nvme_fcreq x%px, "
1998 "lpfc_ncmd %px xri x%x\n",
1999 pnvme_fcreq, lpfc_nbuf,
2000 nvmereq_wqe->sli4_xritag);
2001 goto out_unlock;
2002 }
2003
2004 abts_buf = __lpfc_sli_get_iocbq(phba);
2005 if (!abts_buf) {
2006 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2007 "6136 No available abort wqes. Skipping "
2008 "Abts req for nvme_fcreq x%px xri x%x\n",
2009 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
2010 goto out_unlock;
2011 }
2012
2013
2014 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
2015
2016 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0);
2017
2018
2019 abts_buf->iocb_flag |= LPFC_IO_NVME;
2020 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
2021 abts_buf->vport = vport;
2022 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
2023 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf);
2024 spin_unlock(&lpfc_nbuf->buf_lock);
2025 spin_unlock_irqrestore(&phba->hbalock, flags);
2026 if (ret_val) {
2027 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2028 "6137 Failed abts issue_wqe with status x%x "
2029 "for nvme_fcreq x%px.\n",
2030 ret_val, pnvme_fcreq);
2031 lpfc_sli_release_iocbq(phba, abts_buf);
2032 return;
2033 }
2034
2035 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
2036 "6138 Transport Abort NVME Request Issued for "
2037 "ox_id x%x on reqtag x%x\n",
2038 nvmereq_wqe->sli4_xritag,
2039 abts_buf->iotag);
2040 return;
2041
2042out_unlock:
2043 spin_unlock(&lpfc_nbuf->buf_lock);
2044 spin_unlock_irqrestore(&phba->hbalock, flags);
2045 return;
2046}
2047
2048
2049static struct nvme_fc_port_template lpfc_nvme_template = {
2050
2051 .localport_delete = lpfc_nvme_localport_delete,
2052 .remoteport_delete = lpfc_nvme_remoteport_delete,
2053 .create_queue = lpfc_nvme_create_queue,
2054 .delete_queue = lpfc_nvme_delete_queue,
2055 .ls_req = lpfc_nvme_ls_req,
2056 .fcp_io = lpfc_nvme_fcp_io_submit,
2057 .ls_abort = lpfc_nvme_ls_abort,
2058 .fcp_abort = lpfc_nvme_fcp_abort,
2059 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
2060
2061 .max_hw_queues = 1,
2062 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2063 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2064 .dma_boundary = 0xFFFFFFFF,
2065
2066
2067
2068
2069 .local_priv_sz = sizeof(struct lpfc_nvme_lport),
2070 .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2071 .lsrqst_priv_sz = 0,
2072 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2073};
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086static struct lpfc_io_buf *
2087lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2088 int idx, int expedite)
2089{
2090 struct lpfc_io_buf *lpfc_ncmd;
2091 struct lpfc_sli4_hdw_queue *qp;
2092 struct sli4_sge *sgl;
2093 struct lpfc_iocbq *pwqeq;
2094 union lpfc_wqe128 *wqe;
2095
2096 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2097
2098 if (lpfc_ncmd) {
2099 pwqeq = &(lpfc_ncmd->cur_iocbq);
2100 wqe = &pwqeq->wqe;
2101
2102
2103
2104
2105 pwqeq->iocb_flag = LPFC_IO_NVME;
2106 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
2107 lpfc_ncmd->start_time = jiffies;
2108 lpfc_ncmd->flags = 0;
2109
2110
2111
2112
2113
2114 sgl = lpfc_ncmd->dma_sgl;
2115 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2116 bf_set(lpfc_sli4_sge_last, sgl, 0);
2117 sgl->word2 = cpu_to_le32(sgl->word2);
2118
2119
2120
2121 memset(wqe, 0, sizeof(union lpfc_wqe));
2122
2123 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2124 atomic_inc(&ndlp->cmd_pending);
2125 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2126 }
2127
2128 } else {
2129 qp = &phba->sli4_hba.hdwq[idx];
2130 qp->empty_io_bufs++;
2131 }
2132
2133 return lpfc_ncmd;
2134}
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146static void
2147lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2148{
2149 struct lpfc_sli4_hdw_queue *qp;
2150 unsigned long iflag = 0;
2151
2152 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2153 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2154
2155 lpfc_ncmd->ndlp = NULL;
2156 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2157
2158 qp = lpfc_ncmd->hdwq;
2159 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2160 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2161 "6310 XB release deferred for "
2162 "ox_id x%x on reqtag x%x\n",
2163 lpfc_ncmd->cur_iocbq.sli4_xritag,
2164 lpfc_ncmd->cur_iocbq.iotag);
2165
2166 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2167 list_add_tail(&lpfc_ncmd->list,
2168 &qp->lpfc_abts_io_buf_list);
2169 qp->abts_nvme_io_bufs++;
2170 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2171 } else
2172 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2173}
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191int
2192lpfc_nvme_create_localport(struct lpfc_vport *vport)
2193{
2194 int ret = 0;
2195 struct lpfc_hba *phba = vport->phba;
2196 struct nvme_fc_port_info nfcp_info;
2197 struct nvme_fc_local_port *localport;
2198 struct lpfc_nvme_lport *lport;
2199
2200
2201
2202
2203 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2204 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2205 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2206 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2207
2208
2209
2210
2211
2212 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2213
2214
2215
2216
2217 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2218
2219 if (!IS_ENABLED(CONFIG_NVME_FC))
2220 return ret;
2221
2222
2223
2224
2225
2226 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2227 &vport->phba->pcidev->dev, &localport);
2228 if (!ret) {
2229 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2230 "6005 Successfully registered local "
2231 "NVME port num %d, localP x%px, private "
2232 "x%px, sg_seg %d\n",
2233 localport->port_num, localport,
2234 localport->private,
2235 lpfc_nvme_template.max_sgl_segments);
2236
2237
2238 lport = (struct lpfc_nvme_lport *)localport->private;
2239 vport->localport = localport;
2240 lport->vport = vport;
2241 vport->nvmei_support = 1;
2242
2243 atomic_set(&lport->xmt_fcp_noxri, 0);
2244 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2245 atomic_set(&lport->xmt_fcp_qdepth, 0);
2246 atomic_set(&lport->xmt_fcp_err, 0);
2247 atomic_set(&lport->xmt_fcp_wqerr, 0);
2248 atomic_set(&lport->xmt_fcp_abort, 0);
2249 atomic_set(&lport->xmt_ls_abort, 0);
2250 atomic_set(&lport->xmt_ls_err, 0);
2251 atomic_set(&lport->cmpl_fcp_xb, 0);
2252 atomic_set(&lport->cmpl_fcp_err, 0);
2253 atomic_set(&lport->cmpl_ls_xb, 0);
2254 atomic_set(&lport->cmpl_ls_err, 0);
2255
2256 atomic_set(&lport->fc4NvmeLsRequests, 0);
2257 atomic_set(&lport->fc4NvmeLsCmpls, 0);
2258 }
2259
2260 return ret;
2261}
2262
2263#if (IS_ENABLED(CONFIG_NVME_FC))
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274static void
2275lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2276 struct lpfc_nvme_lport *lport,
2277 struct completion *lport_unreg_cmp)
2278{
2279 u32 wait_tmo;
2280 int ret, i, pending = 0;
2281 struct lpfc_sli_ring *pring;
2282 struct lpfc_hba *phba = vport->phba;
2283
2284
2285
2286
2287
2288 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2289 while (true) {
2290 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2291 if (unlikely(!ret)) {
2292 pending = 0;
2293 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2294 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2295 if (!pring)
2296 continue;
2297 if (pring->txcmplq_cnt)
2298 pending += pring->txcmplq_cnt;
2299 }
2300 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2301 "6176 Lport x%px Localport x%px wait "
2302 "timed out. Pending %d. Renewing.\n",
2303 lport, vport->localport, pending);
2304 continue;
2305 }
2306 break;
2307 }
2308 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2309 "6177 Lport x%px Localport x%px Complete Success\n",
2310 lport, vport->localport);
2311}
2312#endif
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324void
2325lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2326{
2327#if (IS_ENABLED(CONFIG_NVME_FC))
2328 struct nvme_fc_local_port *localport;
2329 struct lpfc_nvme_lport *lport;
2330 int ret;
2331 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2332
2333 if (vport->nvmei_support == 0)
2334 return;
2335
2336 localport = vport->localport;
2337 lport = (struct lpfc_nvme_lport *)localport->private;
2338
2339 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2340 "6011 Destroying NVME localport x%px\n",
2341 localport);
2342
2343
2344
2345
2346 lport->lport_unreg_cmp = &lport_unreg_cmp;
2347 ret = nvme_fc_unregister_localport(localport);
2348
2349
2350
2351
2352 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2353 vport->localport = NULL;
2354
2355
2356
2357
2358
2359 vport->nvmei_support = 0;
2360 if (ret == 0) {
2361 lpfc_printf_vlog(vport,
2362 KERN_INFO, LOG_NVME_DISC,
2363 "6009 Unregistered lport Success\n");
2364 } else {
2365 lpfc_printf_vlog(vport,
2366 KERN_INFO, LOG_NVME_DISC,
2367 "6010 Unregistered lport "
2368 "Failed, status x%x\n",
2369 ret);
2370 }
2371#endif
2372}
2373
2374void
2375lpfc_nvme_update_localport(struct lpfc_vport *vport)
2376{
2377#if (IS_ENABLED(CONFIG_NVME_FC))
2378 struct nvme_fc_local_port *localport;
2379 struct lpfc_nvme_lport *lport;
2380
2381 localport = vport->localport;
2382 if (!localport) {
2383 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2384 "6710 Update NVME fail. No localport\n");
2385 return;
2386 }
2387 lport = (struct lpfc_nvme_lport *)localport->private;
2388 if (!lport) {
2389 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2390 "6171 Update NVME fail. localP x%px, No lport\n",
2391 localport);
2392 return;
2393 }
2394 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2395 "6012 Update NVME lport x%px did x%x\n",
2396 localport, vport->fc_myDID);
2397
2398 localport->port_id = vport->fc_myDID;
2399 if (localport->port_id == 0)
2400 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2401 else
2402 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2403
2404 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2405 "6030 bound lport x%px to DID x%06x\n",
2406 lport, localport->port_id);
2407#endif
2408}
2409
2410int
2411lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2412{
2413#if (IS_ENABLED(CONFIG_NVME_FC))
2414 int ret = 0;
2415 struct nvme_fc_local_port *localport;
2416 struct lpfc_nvme_lport *lport;
2417 struct lpfc_nvme_rport *rport;
2418 struct lpfc_nvme_rport *oldrport;
2419 struct nvme_fc_remote_port *remote_port;
2420 struct nvme_fc_port_info rpinfo;
2421 struct lpfc_nodelist *prev_ndlp = NULL;
2422 struct fc_rport *srport = ndlp->rport;
2423
2424 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2425 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2426 ndlp->nlp_DID, ndlp->nlp_type);
2427
2428 localport = vport->localport;
2429 if (!localport)
2430 return 0;
2431
2432 lport = (struct lpfc_nvme_lport *)localport->private;
2433
2434
2435
2436
2437
2438
2439
2440 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2441 rpinfo.port_id = ndlp->nlp_DID;
2442 if (ndlp->nlp_type & NLP_NVME_TARGET)
2443 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2444 if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2445 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2446
2447 if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2448 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2449
2450 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2451 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2452 if (srport)
2453 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2454 else
2455 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2456
2457 spin_lock_irq(&vport->phba->hbalock);
2458 oldrport = lpfc_ndlp_get_nrport(ndlp);
2459 if (oldrport) {
2460 prev_ndlp = oldrport->ndlp;
2461 spin_unlock_irq(&vport->phba->hbalock);
2462 } else {
2463 spin_unlock_irq(&vport->phba->hbalock);
2464 lpfc_nlp_get(ndlp);
2465 }
2466
2467 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2468 if (!ret) {
2469
2470
2471
2472
2473
2474
2475
2476 spin_lock_irq(&vport->phba->hbalock);
2477 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2478 spin_unlock_irq(&vport->phba->hbalock);
2479 rport = remote_port->private;
2480 if (oldrport) {
2481
2482
2483
2484
2485
2486 spin_lock_irq(&vport->phba->hbalock);
2487 ndlp->nrport = NULL;
2488 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2489 spin_unlock_irq(&vport->phba->hbalock);
2490 rport->ndlp = NULL;
2491 rport->remoteport = NULL;
2492
2493
2494
2495
2496
2497 if (prev_ndlp && prev_ndlp != ndlp) {
2498 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
2499 (!prev_ndlp->nrport))
2500 lpfc_nlp_put(prev_ndlp);
2501 }
2502 }
2503
2504
2505 rport->remoteport = remote_port;
2506 rport->lport = lport;
2507 rport->ndlp = ndlp;
2508 spin_lock_irq(&vport->phba->hbalock);
2509 ndlp->nrport = rport;
2510 spin_unlock_irq(&vport->phba->hbalock);
2511 lpfc_printf_vlog(vport, KERN_INFO,
2512 LOG_NVME_DISC | LOG_NODE,
2513 "6022 Bind lport x%px to remoteport x%px "
2514 "rport x%px WWNN 0x%llx, "
2515 "Rport WWPN 0x%llx DID "
2516 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2517 lport, remote_port, rport,
2518 rpinfo.node_name, rpinfo.port_name,
2519 rpinfo.port_id, rpinfo.port_role,
2520 ndlp, prev_ndlp);
2521 } else {
2522 lpfc_printf_vlog(vport, KERN_ERR,
2523 LOG_TRACE_EVENT,
2524 "6031 RemotePort Registration failed "
2525 "err: %d, DID x%06x\n",
2526 ret, ndlp->nlp_DID);
2527 }
2528
2529 return ret;
2530#else
2531 return 0;
2532#endif
2533}
2534
2535
2536
2537
2538
2539
2540
2541
2542void
2543lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2544{
2545#if (IS_ENABLED(CONFIG_NVME_FC))
2546 struct lpfc_nvme_rport *nrport;
2547 struct nvme_fc_remote_port *remoteport = NULL;
2548
2549 spin_lock_irq(&vport->phba->hbalock);
2550 nrport = lpfc_ndlp_get_nrport(ndlp);
2551 if (nrport)
2552 remoteport = nrport->remoteport;
2553 spin_unlock_irq(&vport->phba->hbalock);
2554
2555 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2556 "6170 Rescan NPort DID x%06x type x%x "
2557 "state x%x nrport x%px remoteport x%px\n",
2558 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2559 nrport, remoteport);
2560
2561 if (!nrport || !remoteport)
2562 goto rescan_exit;
2563
2564
2565 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2566 ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2567 nvme_fc_rescan_remoteport(remoteport);
2568
2569 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2570 "6172 NVME rescanned DID x%06x "
2571 "port_state x%x\n",
2572 ndlp->nlp_DID, remoteport->port_state);
2573 }
2574 return;
2575 rescan_exit:
2576 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2577 "6169 Skip NVME Rport Rescan, NVME remoteport "
2578 "unregistered\n");
2579#endif
2580}
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594void
2595lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2596{
2597#if (IS_ENABLED(CONFIG_NVME_FC))
2598 int ret;
2599 struct nvme_fc_local_port *localport;
2600 struct lpfc_nvme_lport *lport;
2601 struct lpfc_nvme_rport *rport;
2602 struct nvme_fc_remote_port *remoteport = NULL;
2603
2604 localport = vport->localport;
2605
2606
2607
2608
2609 if (!localport)
2610 return;
2611
2612 lport = (struct lpfc_nvme_lport *)localport->private;
2613 if (!lport)
2614 goto input_err;
2615
2616 spin_lock_irq(&vport->phba->hbalock);
2617 rport = lpfc_ndlp_get_nrport(ndlp);
2618 if (rport)
2619 remoteport = rport->remoteport;
2620 spin_unlock_irq(&vport->phba->hbalock);
2621 if (!remoteport)
2622 goto input_err;
2623
2624 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2625 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2626 "port_id x%06x, portstate x%x port type x%x\n",
2627 remoteport, remoteport->port_name,
2628 remoteport->port_id, remoteport->port_state,
2629 ndlp->nlp_type);
2630
2631
2632
2633
2634
2635 if (ndlp->nlp_type & NLP_NVME_TARGET) {
2636
2637
2638
2639 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
2640
2641
2642
2643
2644
2645
2646 if (vport->load_flag & FC_UNLOADING)
2647 (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2648
2649 ret = nvme_fc_unregister_remoteport(remoteport);
2650 if (ret != 0) {
2651 lpfc_nlp_put(ndlp);
2652 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2653 "6167 NVME unregister failed %d "
2654 "port_state x%x\n",
2655 ret, remoteport->port_state);
2656 }
2657 }
2658 return;
2659
2660 input_err:
2661#endif
2662 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2663 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2664 vport->localport, ndlp->rport, ndlp->nlp_DID);
2665}
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677void
2678lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2679 struct sli4_wcqe_xri_aborted *axri,
2680 struct lpfc_io_buf *lpfc_ncmd)
2681{
2682 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2683 struct nvmefc_fcp_req *nvme_cmd = NULL;
2684 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2685
2686
2687 if (ndlp)
2688 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2689
2690 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2691 "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2692 "xri released\n",
2693 lpfc_ncmd->nvmeCmd, xri,
2694 lpfc_ncmd->cur_iocbq.iotag);
2695
2696
2697
2698
2699
2700 if (lpfc_ncmd->nvmeCmd) {
2701 nvme_cmd = lpfc_ncmd->nvmeCmd;
2702 nvme_cmd->done(nvme_cmd);
2703 lpfc_ncmd->nvmeCmd = NULL;
2704 }
2705 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2706}
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718void
2719lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2720{
2721 struct lpfc_sli_ring *pring;
2722 u32 i, wait_cnt = 0;
2723
2724 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2725 return;
2726
2727
2728
2729
2730 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2731 if (!phba->sli4_hba.hdwq[i].io_wq)
2732 continue;
2733 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2734
2735 if (!pring)
2736 continue;
2737
2738
2739 while (!list_empty(&pring->txcmplq)) {
2740 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2741 wait_cnt++;
2742
2743
2744
2745
2746 if ((wait_cnt % 1000) == 0) {
2747 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2748 "6178 NVME IO not empty, "
2749 "cnt %d\n", wait_cnt);
2750 }
2751 }
2752 }
2753}
2754
2755void
2756lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn)
2757{
2758#if (IS_ENABLED(CONFIG_NVME_FC))
2759 struct lpfc_io_buf *lpfc_ncmd;
2760 struct nvmefc_fcp_req *nCmd;
2761 struct lpfc_nvme_fcpreq_priv *freqpriv;
2762
2763 if (!pwqeIn->context1) {
2764 lpfc_sli_release_iocbq(phba, pwqeIn);
2765 return;
2766 }
2767
2768 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2769 CMD_ABORT_XRI_CX) {
2770 lpfc_sli_release_iocbq(phba, pwqeIn);
2771 return;
2772 }
2773 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2774
2775 spin_lock(&lpfc_ncmd->buf_lock);
2776 if (!lpfc_ncmd->nvmeCmd) {
2777 spin_unlock(&lpfc_ncmd->buf_lock);
2778 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2779 return;
2780 }
2781
2782 nCmd = lpfc_ncmd->nvmeCmd;
2783 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2784 "6194 NVME Cancel xri %x\n",
2785 lpfc_ncmd->cur_iocbq.sli4_xritag);
2786
2787 nCmd->transferred_length = 0;
2788 nCmd->rcv_rsplen = 0;
2789 nCmd->status = NVME_SC_INTERNAL;
2790 freqpriv = nCmd->private;
2791 freqpriv->nvme_buf = NULL;
2792 lpfc_ncmd->nvmeCmd = NULL;
2793
2794 spin_unlock(&lpfc_ncmd->buf_lock);
2795 nCmd->done(nCmd);
2796
2797
2798 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2799#endif
2800}
2801