1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18#include <linux/module.h>
19#include <linux/slab.h>
20#include <linux/blk-mq.h>
21#include <linux/parser.h>
22#include <linux/random.h>
23#include <uapi/scsi/fc/fc_fs.h>
24#include <uapi/scsi/fc/fc_els.h>
25
26#include "nvmet.h"
27#include <linux/nvme-fc-driver.h>
28#include <linux/nvme-fc.h>
29
30
31
32
33
34#define NVMET_LS_CTX_COUNT 4
35
36
37#define NVME_FC_MAX_LS_BUFFER_SIZE 2048
38
39struct nvmet_fc_tgtport;
40struct nvmet_fc_tgt_assoc;
41
42struct nvmet_fc_ls_iod {
43 struct nvmefc_tgt_ls_req *lsreq;
44 struct nvmefc_tgt_fcp_req *fcpreq;
45
46 struct list_head ls_list;
47
48 struct nvmet_fc_tgtport *tgtport;
49 struct nvmet_fc_tgt_assoc *assoc;
50
51 u8 *rqstbuf;
52 u8 *rspbuf;
53 u16 rqstdatalen;
54 dma_addr_t rspdma;
55
56 struct scatterlist sg[2];
57
58 struct work_struct work;
59} __aligned(sizeof(unsigned long long));
60
61#define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
62#define NVMET_FC_MAX_XFR_SGENTS (NVMET_FC_MAX_SEQ_LENGTH / PAGE_SIZE)
63
64enum nvmet_fcp_datadir {
65 NVMET_FCP_NODATA,
66 NVMET_FCP_WRITE,
67 NVMET_FCP_READ,
68 NVMET_FCP_ABORTED,
69};
70
71struct nvmet_fc_fcp_iod {
72 struct nvmefc_tgt_fcp_req *fcpreq;
73
74 struct nvme_fc_cmd_iu cmdiubuf;
75 struct nvme_fc_ersp_iu rspiubuf;
76 dma_addr_t rspdma;
77 struct scatterlist *data_sg;
78 int data_sg_cnt;
79 u32 offset;
80 enum nvmet_fcp_datadir io_dir;
81 bool active;
82 bool abort;
83 bool aborted;
84 bool writedataactive;
85 spinlock_t flock;
86
87 struct nvmet_req req;
88 struct work_struct work;
89 struct work_struct done_work;
90 struct work_struct defer_work;
91
92 struct nvmet_fc_tgtport *tgtport;
93 struct nvmet_fc_tgt_queue *queue;
94
95 struct list_head fcp_list;
96};
97
98struct nvmet_fc_tgtport {
99
100 struct nvmet_fc_target_port fc_target_port;
101
102 struct list_head tgt_list;
103 struct device *dev;
104 struct nvmet_fc_target_template *ops;
105
106 struct nvmet_fc_ls_iod *iod;
107 spinlock_t lock;
108 struct list_head ls_list;
109 struct list_head ls_busylist;
110 struct list_head assoc_list;
111 struct ida assoc_cnt;
112 struct nvmet_port *port;
113 struct kref ref;
114 u32 max_sg_cnt;
115};
116
117struct nvmet_fc_defer_fcp_req {
118 struct list_head req_list;
119 struct nvmefc_tgt_fcp_req *fcp_req;
120};
121
122struct nvmet_fc_tgt_queue {
123 bool ninetypercent;
124 u16 qid;
125 u16 sqsize;
126 u16 ersp_ratio;
127 __le16 sqhd;
128 int cpu;
129 atomic_t connected;
130 atomic_t sqtail;
131 atomic_t zrspcnt;
132 atomic_t rsn;
133 spinlock_t qlock;
134 struct nvmet_port *port;
135 struct nvmet_cq nvme_cq;
136 struct nvmet_sq nvme_sq;
137 struct nvmet_fc_tgt_assoc *assoc;
138 struct nvmet_fc_fcp_iod *fod;
139 struct list_head fod_list;
140 struct list_head pending_cmd_list;
141 struct list_head avail_defer_list;
142 struct workqueue_struct *work_q;
143 struct kref ref;
144} __aligned(sizeof(unsigned long long));
145
146struct nvmet_fc_tgt_assoc {
147 u64 association_id;
148 u32 a_id;
149 struct nvmet_fc_tgtport *tgtport;
150 struct list_head a_list;
151 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
152 struct kref ref;
153 struct work_struct del_work;
154};
155
156
157static inline int
158nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
159{
160 return (iodptr - iodptr->tgtport->iod);
161}
162
163static inline int
164nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
165{
166 return (fodptr - fodptr->queue->fod);
167}
168
169
170
171
172
173
174
175
176
177
178
179
180#define BYTES_FOR_QID sizeof(u16)
181#define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
182#define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
183
184static inline u64
185nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
186{
187 return (assoc->association_id | qid);
188}
189
190static inline u64
191nvmet_fc_getassociationid(u64 connectionid)
192{
193 return connectionid & ~NVMET_FC_QUEUEID_MASK;
194}
195
196static inline u16
197nvmet_fc_getqueueid(u64 connectionid)
198{
199 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
200}
201
202static inline struct nvmet_fc_tgtport *
203targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
204{
205 return container_of(targetport, struct nvmet_fc_tgtport,
206 fc_target_port);
207}
208
209static inline struct nvmet_fc_fcp_iod *
210nvmet_req_to_fod(struct nvmet_req *nvme_req)
211{
212 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
213}
214
215
216
217
218
219static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
220
221static LIST_HEAD(nvmet_fc_target_list);
222static DEFINE_IDA(nvmet_fc_tgtport_cnt);
223
224
225static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
226static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
227static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work);
228static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
229static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
230static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
231static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
232static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
233static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
234static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
235static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
236 struct nvmet_fc_fcp_iod *fod);
237static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258static inline dma_addr_t
259fc_dma_map_single(struct device *dev, void *ptr, size_t size,
260 enum dma_data_direction dir)
261{
262 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
263}
264
265static inline int
266fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
267{
268 return dev ? dma_mapping_error(dev, dma_addr) : 0;
269}
270
271static inline void
272fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
273 enum dma_data_direction dir)
274{
275 if (dev)
276 dma_unmap_single(dev, addr, size, dir);
277}
278
279static inline void
280fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
281 enum dma_data_direction dir)
282{
283 if (dev)
284 dma_sync_single_for_cpu(dev, addr, size, dir);
285}
286
287static inline void
288fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
289 enum dma_data_direction dir)
290{
291 if (dev)
292 dma_sync_single_for_device(dev, addr, size, dir);
293}
294
295
296static int
297fc_map_sg(struct scatterlist *sg, int nents)
298{
299 struct scatterlist *s;
300 int i;
301
302 WARN_ON(nents == 0 || sg[0].length == 0);
303
304 for_each_sg(sg, s, nents, i) {
305 s->dma_address = 0L;
306#ifdef CONFIG_NEED_SG_DMA_LENGTH
307 s->dma_length = s->length;
308#endif
309 }
310 return nents;
311}
312
313static inline int
314fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
315 enum dma_data_direction dir)
316{
317 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
318}
319
320static inline void
321fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
322 enum dma_data_direction dir)
323{
324 if (dev)
325 dma_unmap_sg(dev, sg, nents, dir);
326}
327
328
329
330
331
332static int
333nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
334{
335 struct nvmet_fc_ls_iod *iod;
336 int i;
337
338 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
339 GFP_KERNEL);
340 if (!iod)
341 return -ENOMEM;
342
343 tgtport->iod = iod;
344
345 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
346 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
347 iod->tgtport = tgtport;
348 list_add_tail(&iod->ls_list, &tgtport->ls_list);
349
350 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
351 GFP_KERNEL);
352 if (!iod->rqstbuf)
353 goto out_fail;
354
355 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
356
357 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
358 NVME_FC_MAX_LS_BUFFER_SIZE,
359 DMA_TO_DEVICE);
360 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
361 goto out_fail;
362 }
363
364 return 0;
365
366out_fail:
367 kfree(iod->rqstbuf);
368 list_del(&iod->ls_list);
369 for (iod--, i--; i >= 0; iod--, i--) {
370 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
371 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
372 kfree(iod->rqstbuf);
373 list_del(&iod->ls_list);
374 }
375
376 kfree(iod);
377
378 return -EFAULT;
379}
380
381static void
382nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
383{
384 struct nvmet_fc_ls_iod *iod = tgtport->iod;
385 int i;
386
387 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
388 fc_dma_unmap_single(tgtport->dev,
389 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
390 DMA_TO_DEVICE);
391 kfree(iod->rqstbuf);
392 list_del(&iod->ls_list);
393 }
394 kfree(tgtport->iod);
395}
396
397static struct nvmet_fc_ls_iod *
398nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
399{
400 struct nvmet_fc_ls_iod *iod;
401 unsigned long flags;
402
403 spin_lock_irqsave(&tgtport->lock, flags);
404 iod = list_first_entry_or_null(&tgtport->ls_list,
405 struct nvmet_fc_ls_iod, ls_list);
406 if (iod)
407 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
408 spin_unlock_irqrestore(&tgtport->lock, flags);
409 return iod;
410}
411
412
413static void
414nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
415 struct nvmet_fc_ls_iod *iod)
416{
417 unsigned long flags;
418
419 spin_lock_irqsave(&tgtport->lock, flags);
420 list_move(&iod->ls_list, &tgtport->ls_list);
421 spin_unlock_irqrestore(&tgtport->lock, flags);
422}
423
424static void
425nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
426 struct nvmet_fc_tgt_queue *queue)
427{
428 struct nvmet_fc_fcp_iod *fod = queue->fod;
429 int i;
430
431 for (i = 0; i < queue->sqsize; fod++, i++) {
432 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
433 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work);
434 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
435 fod->tgtport = tgtport;
436 fod->queue = queue;
437 fod->active = false;
438 fod->abort = false;
439 fod->aborted = false;
440 fod->fcpreq = NULL;
441 list_add_tail(&fod->fcp_list, &queue->fod_list);
442 spin_lock_init(&fod->flock);
443
444 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
445 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
446 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
447 list_del(&fod->fcp_list);
448 for (fod--, i--; i >= 0; fod--, i--) {
449 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
450 sizeof(fod->rspiubuf),
451 DMA_TO_DEVICE);
452 fod->rspdma = 0L;
453 list_del(&fod->fcp_list);
454 }
455
456 return;
457 }
458 }
459}
460
461static void
462nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
463 struct nvmet_fc_tgt_queue *queue)
464{
465 struct nvmet_fc_fcp_iod *fod = queue->fod;
466 int i;
467
468 for (i = 0; i < queue->sqsize; fod++, i++) {
469 if (fod->rspdma)
470 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
471 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
472 }
473}
474
475static struct nvmet_fc_fcp_iod *
476nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
477{
478 struct nvmet_fc_fcp_iod *fod;
479
480 lockdep_assert_held(&queue->qlock);
481
482 fod = list_first_entry_or_null(&queue->fod_list,
483 struct nvmet_fc_fcp_iod, fcp_list);
484 if (fod) {
485 list_del(&fod->fcp_list);
486 fod->active = true;
487
488
489
490
491
492 }
493 return fod;
494}
495
496
497static void
498nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
499 struct nvmet_fc_tgt_queue *queue,
500 struct nvmefc_tgt_fcp_req *fcpreq)
501{
502 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
503
504
505
506
507
508 fcpreq->hwqid = queue->qid ?
509 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
510
511 if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR)
512 queue_work_on(queue->cpu, queue->work_q, &fod->work);
513 else
514 nvmet_fc_handle_fcp_rqst(tgtport, fod);
515}
516
517static void
518nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
519{
520 struct nvmet_fc_fcp_iod *fod =
521 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
522
523
524 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
525
526}
527
528static void
529nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
530 struct nvmet_fc_fcp_iod *fod)
531{
532 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
533 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
534 struct nvmet_fc_defer_fcp_req *deferfcp;
535 unsigned long flags;
536
537 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
538 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
539
540 fcpreq->nvmet_fc_private = NULL;
541
542 fod->active = false;
543 fod->abort = false;
544 fod->aborted = false;
545 fod->writedataactive = false;
546 fod->fcpreq = NULL;
547
548 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
549
550
551 nvmet_fc_tgt_q_put(queue);
552
553 spin_lock_irqsave(&queue->qlock, flags);
554 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
555 struct nvmet_fc_defer_fcp_req, req_list);
556 if (!deferfcp) {
557 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
558 spin_unlock_irqrestore(&queue->qlock, flags);
559 return;
560 }
561
562
563 list_del(&deferfcp->req_list);
564
565 fcpreq = deferfcp->fcp_req;
566
567
568 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
569
570 spin_unlock_irqrestore(&queue->qlock, flags);
571
572
573 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
574
575
576 fcpreq->rspaddr = NULL;
577 fcpreq->rsplen = 0;
578 fcpreq->nvmet_fc_private = fod;
579 fod->fcpreq = fcpreq;
580 fod->active = true;
581
582
583 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
584
585
586
587
588
589
590 queue_work(queue->work_q, &fod->defer_work);
591}
592
593static int
594nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
595{
596 int cpu, idx, cnt;
597
598 if (tgtport->ops->max_hw_queues == 1)
599 return WORK_CPU_UNBOUND;
600
601
602 idx = !qid ? 0 : (qid - 1) % num_active_cpus();
603
604
605 for (cpu = 0, cnt = 0; ; ) {
606 if (cpu_active(cpu)) {
607 if (cnt == idx)
608 break;
609 cnt++;
610 }
611 cpu = (cpu + 1) % num_possible_cpus();
612 }
613
614 return cpu;
615}
616
617static struct nvmet_fc_tgt_queue *
618nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
619 u16 qid, u16 sqsize)
620{
621 struct nvmet_fc_tgt_queue *queue;
622 unsigned long flags;
623 int ret;
624
625 if (qid > NVMET_NR_QUEUES)
626 return NULL;
627
628 queue = kzalloc((sizeof(*queue) +
629 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
630 GFP_KERNEL);
631 if (!queue)
632 return NULL;
633
634 if (!nvmet_fc_tgt_a_get(assoc))
635 goto out_free_queue;
636
637 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
638 assoc->tgtport->fc_target_port.port_num,
639 assoc->a_id, qid);
640 if (!queue->work_q)
641 goto out_a_put;
642
643 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
644 queue->qid = qid;
645 queue->sqsize = sqsize;
646 queue->assoc = assoc;
647 queue->port = assoc->tgtport->port;
648 queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
649 INIT_LIST_HEAD(&queue->fod_list);
650 INIT_LIST_HEAD(&queue->avail_defer_list);
651 INIT_LIST_HEAD(&queue->pending_cmd_list);
652 atomic_set(&queue->connected, 0);
653 atomic_set(&queue->sqtail, 0);
654 atomic_set(&queue->rsn, 1);
655 atomic_set(&queue->zrspcnt, 0);
656 spin_lock_init(&queue->qlock);
657 kref_init(&queue->ref);
658
659 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
660
661 ret = nvmet_sq_init(&queue->nvme_sq);
662 if (ret)
663 goto out_fail_iodlist;
664
665 WARN_ON(assoc->queues[qid]);
666 spin_lock_irqsave(&assoc->tgtport->lock, flags);
667 assoc->queues[qid] = queue;
668 spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
669
670 return queue;
671
672out_fail_iodlist:
673 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
674 destroy_workqueue(queue->work_q);
675out_a_put:
676 nvmet_fc_tgt_a_put(assoc);
677out_free_queue:
678 kfree(queue);
679 return NULL;
680}
681
682
683static void
684nvmet_fc_tgt_queue_free(struct kref *ref)
685{
686 struct nvmet_fc_tgt_queue *queue =
687 container_of(ref, struct nvmet_fc_tgt_queue, ref);
688 unsigned long flags;
689
690 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
691 queue->assoc->queues[queue->qid] = NULL;
692 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
693
694 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
695
696 nvmet_fc_tgt_a_put(queue->assoc);
697
698 destroy_workqueue(queue->work_q);
699
700 kfree(queue);
701}
702
703static void
704nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
705{
706 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
707}
708
709static int
710nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
711{
712 return kref_get_unless_zero(&queue->ref);
713}
714
715
716static void
717nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
718{
719 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
720 struct nvmet_fc_fcp_iod *fod = queue->fod;
721 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
722 unsigned long flags;
723 int i, writedataactive;
724 bool disconnect;
725
726 disconnect = atomic_xchg(&queue->connected, 0);
727
728 spin_lock_irqsave(&queue->qlock, flags);
729
730 for (i = 0; i < queue->sqsize; fod++, i++) {
731 if (fod->active) {
732 spin_lock(&fod->flock);
733 fod->abort = true;
734 writedataactive = fod->writedataactive;
735 spin_unlock(&fod->flock);
736
737
738
739
740
741 if (writedataactive) {
742 spin_lock(&fod->flock);
743 fod->aborted = true;
744 spin_unlock(&fod->flock);
745 tgtport->ops->fcp_abort(
746 &tgtport->fc_target_port, fod->fcpreq);
747 }
748 }
749 }
750
751
752 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
753 req_list) {
754 list_del(&deferfcp->req_list);
755 kfree(deferfcp);
756 }
757
758 for (;;) {
759 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
760 struct nvmet_fc_defer_fcp_req, req_list);
761 if (!deferfcp)
762 break;
763
764 list_del(&deferfcp->req_list);
765 spin_unlock_irqrestore(&queue->qlock, flags);
766
767 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
768 deferfcp->fcp_req);
769
770 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
771 deferfcp->fcp_req);
772
773 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
774 deferfcp->fcp_req);
775
776
777 nvmet_fc_tgt_q_put(queue);
778
779 kfree(deferfcp);
780
781 spin_lock_irqsave(&queue->qlock, flags);
782 }
783 spin_unlock_irqrestore(&queue->qlock, flags);
784
785 flush_workqueue(queue->work_q);
786
787 if (disconnect)
788 nvmet_sq_destroy(&queue->nvme_sq);
789
790 nvmet_fc_tgt_q_put(queue);
791}
792
793static struct nvmet_fc_tgt_queue *
794nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
795 u64 connection_id)
796{
797 struct nvmet_fc_tgt_assoc *assoc;
798 struct nvmet_fc_tgt_queue *queue;
799 u64 association_id = nvmet_fc_getassociationid(connection_id);
800 u16 qid = nvmet_fc_getqueueid(connection_id);
801 unsigned long flags;
802
803 if (qid > NVMET_NR_QUEUES)
804 return NULL;
805
806 spin_lock_irqsave(&tgtport->lock, flags);
807 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
808 if (association_id == assoc->association_id) {
809 queue = assoc->queues[qid];
810 if (queue &&
811 (!atomic_read(&queue->connected) ||
812 !nvmet_fc_tgt_q_get(queue)))
813 queue = NULL;
814 spin_unlock_irqrestore(&tgtport->lock, flags);
815 return queue;
816 }
817 }
818 spin_unlock_irqrestore(&tgtport->lock, flags);
819 return NULL;
820}
821
822static void
823nvmet_fc_delete_assoc(struct work_struct *work)
824{
825 struct nvmet_fc_tgt_assoc *assoc =
826 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
827
828 nvmet_fc_delete_target_assoc(assoc);
829 nvmet_fc_tgt_a_put(assoc);
830}
831
832static struct nvmet_fc_tgt_assoc *
833nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
834{
835 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
836 unsigned long flags;
837 u64 ran;
838 int idx;
839 bool needrandom = true;
840
841 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
842 if (!assoc)
843 return NULL;
844
845 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
846 if (idx < 0)
847 goto out_free_assoc;
848
849 if (!nvmet_fc_tgtport_get(tgtport))
850 goto out_ida_put;
851
852 assoc->tgtport = tgtport;
853 assoc->a_id = idx;
854 INIT_LIST_HEAD(&assoc->a_list);
855 kref_init(&assoc->ref);
856 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc);
857
858 while (needrandom) {
859 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
860 ran = ran << BYTES_FOR_QID_SHIFT;
861
862 spin_lock_irqsave(&tgtport->lock, flags);
863 needrandom = false;
864 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
865 if (ran == tmpassoc->association_id) {
866 needrandom = true;
867 break;
868 }
869 if (!needrandom) {
870 assoc->association_id = ran;
871 list_add_tail(&assoc->a_list, &tgtport->assoc_list);
872 }
873 spin_unlock_irqrestore(&tgtport->lock, flags);
874 }
875
876 return assoc;
877
878out_ida_put:
879 ida_simple_remove(&tgtport->assoc_cnt, idx);
880out_free_assoc:
881 kfree(assoc);
882 return NULL;
883}
884
885static void
886nvmet_fc_target_assoc_free(struct kref *ref)
887{
888 struct nvmet_fc_tgt_assoc *assoc =
889 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
890 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
891 unsigned long flags;
892
893 spin_lock_irqsave(&tgtport->lock, flags);
894 list_del(&assoc->a_list);
895 spin_unlock_irqrestore(&tgtport->lock, flags);
896 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
897 kfree(assoc);
898 nvmet_fc_tgtport_put(tgtport);
899}
900
901static void
902nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
903{
904 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
905}
906
907static int
908nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
909{
910 return kref_get_unless_zero(&assoc->ref);
911}
912
913static void
914nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
915{
916 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
917 struct nvmet_fc_tgt_queue *queue;
918 unsigned long flags;
919 int i;
920
921 spin_lock_irqsave(&tgtport->lock, flags);
922 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
923 queue = assoc->queues[i];
924 if (queue) {
925 if (!nvmet_fc_tgt_q_get(queue))
926 continue;
927 spin_unlock_irqrestore(&tgtport->lock, flags);
928 nvmet_fc_delete_target_queue(queue);
929 nvmet_fc_tgt_q_put(queue);
930 spin_lock_irqsave(&tgtport->lock, flags);
931 }
932 }
933 spin_unlock_irqrestore(&tgtport->lock, flags);
934
935 nvmet_fc_tgt_a_put(assoc);
936}
937
938static struct nvmet_fc_tgt_assoc *
939nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
940 u64 association_id)
941{
942 struct nvmet_fc_tgt_assoc *assoc;
943 struct nvmet_fc_tgt_assoc *ret = NULL;
944 unsigned long flags;
945
946 spin_lock_irqsave(&tgtport->lock, flags);
947 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
948 if (association_id == assoc->association_id) {
949 ret = assoc;
950 nvmet_fc_tgt_a_get(assoc);
951 break;
952 }
953 }
954 spin_unlock_irqrestore(&tgtport->lock, flags);
955
956 return ret;
957}
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977int
978nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
979 struct nvmet_fc_target_template *template,
980 struct device *dev,
981 struct nvmet_fc_target_port **portptr)
982{
983 struct nvmet_fc_tgtport *newrec;
984 unsigned long flags;
985 int ret, idx;
986
987 if (!template->xmt_ls_rsp || !template->fcp_op ||
988 !template->fcp_abort ||
989 !template->fcp_req_release || !template->targetport_delete ||
990 !template->max_hw_queues || !template->max_sgl_segments ||
991 !template->max_dif_sgl_segments || !template->dma_boundary) {
992 ret = -EINVAL;
993 goto out_regtgt_failed;
994 }
995
996 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
997 GFP_KERNEL);
998 if (!newrec) {
999 ret = -ENOMEM;
1000 goto out_regtgt_failed;
1001 }
1002
1003 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
1004 if (idx < 0) {
1005 ret = -ENOSPC;
1006 goto out_fail_kfree;
1007 }
1008
1009 if (!get_device(dev) && dev) {
1010 ret = -ENODEV;
1011 goto out_ida_put;
1012 }
1013
1014 newrec->fc_target_port.node_name = pinfo->node_name;
1015 newrec->fc_target_port.port_name = pinfo->port_name;
1016 newrec->fc_target_port.private = &newrec[1];
1017 newrec->fc_target_port.port_id = pinfo->port_id;
1018 newrec->fc_target_port.port_num = idx;
1019 INIT_LIST_HEAD(&newrec->tgt_list);
1020 newrec->dev = dev;
1021 newrec->ops = template;
1022 spin_lock_init(&newrec->lock);
1023 INIT_LIST_HEAD(&newrec->ls_list);
1024 INIT_LIST_HEAD(&newrec->ls_busylist);
1025 INIT_LIST_HEAD(&newrec->assoc_list);
1026 kref_init(&newrec->ref);
1027 ida_init(&newrec->assoc_cnt);
1028 newrec->max_sg_cnt = min_t(u32, NVMET_FC_MAX_XFR_SGENTS,
1029 template->max_sgl_segments);
1030
1031 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1032 if (ret) {
1033 ret = -ENOMEM;
1034 goto out_free_newrec;
1035 }
1036
1037 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1038 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1039 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1040
1041 *portptr = &newrec->fc_target_port;
1042 return 0;
1043
1044out_free_newrec:
1045 put_device(dev);
1046out_ida_put:
1047 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1048out_fail_kfree:
1049 kfree(newrec);
1050out_regtgt_failed:
1051 *portptr = NULL;
1052 return ret;
1053}
1054EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1055
1056
1057static void
1058nvmet_fc_free_tgtport(struct kref *ref)
1059{
1060 struct nvmet_fc_tgtport *tgtport =
1061 container_of(ref, struct nvmet_fc_tgtport, ref);
1062 struct device *dev = tgtport->dev;
1063 unsigned long flags;
1064
1065 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1066 list_del(&tgtport->tgt_list);
1067 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1068
1069 nvmet_fc_free_ls_iodlist(tgtport);
1070
1071
1072 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1073
1074 ida_simple_remove(&nvmet_fc_tgtport_cnt,
1075 tgtport->fc_target_port.port_num);
1076
1077 ida_destroy(&tgtport->assoc_cnt);
1078
1079 kfree(tgtport);
1080
1081 put_device(dev);
1082}
1083
1084static void
1085nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1086{
1087 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1088}
1089
1090static int
1091nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1092{
1093 return kref_get_unless_zero(&tgtport->ref);
1094}
1095
1096static void
1097__nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1098{
1099 struct nvmet_fc_tgt_assoc *assoc, *next;
1100 unsigned long flags;
1101
1102 spin_lock_irqsave(&tgtport->lock, flags);
1103 list_for_each_entry_safe(assoc, next,
1104 &tgtport->assoc_list, a_list) {
1105 if (!nvmet_fc_tgt_a_get(assoc))
1106 continue;
1107 spin_unlock_irqrestore(&tgtport->lock, flags);
1108 nvmet_fc_delete_target_assoc(assoc);
1109 nvmet_fc_tgt_a_put(assoc);
1110 spin_lock_irqsave(&tgtport->lock, flags);
1111 }
1112 spin_unlock_irqrestore(&tgtport->lock, flags);
1113}
1114
1115
1116
1117
1118static void
1119nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1120{
1121 struct nvmet_fc_tgtport *tgtport, *next;
1122 struct nvmet_fc_tgt_assoc *assoc;
1123 struct nvmet_fc_tgt_queue *queue;
1124 unsigned long flags;
1125 bool found_ctrl = false;
1126
1127
1128 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1129 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1130 tgt_list) {
1131 if (!nvmet_fc_tgtport_get(tgtport))
1132 continue;
1133 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1134
1135 spin_lock_irqsave(&tgtport->lock, flags);
1136 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1137 queue = assoc->queues[0];
1138 if (queue && queue->nvme_sq.ctrl == ctrl) {
1139 if (nvmet_fc_tgt_a_get(assoc))
1140 found_ctrl = true;
1141 break;
1142 }
1143 }
1144 spin_unlock_irqrestore(&tgtport->lock, flags);
1145
1146 nvmet_fc_tgtport_put(tgtport);
1147
1148 if (found_ctrl) {
1149 schedule_work(&assoc->del_work);
1150 return;
1151 }
1152
1153 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1154 }
1155 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1156}
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169int
1170nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1171{
1172 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1173
1174
1175 __nvmet_fc_free_assocs(tgtport);
1176
1177 nvmet_fc_tgtport_put(tgtport);
1178
1179 return 0;
1180}
1181EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1182
1183
1184
1185
1186
1187static void
1188nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1189{
1190 struct fcnvme_ls_acc_hdr *acc = buf;
1191
1192 acc->w0.ls_cmd = ls_cmd;
1193 acc->desc_list_len = desc_len;
1194 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1195 acc->rqst.desc_len =
1196 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1197 acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1198}
1199
1200static int
1201nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1202 u8 reason, u8 explanation, u8 vendor)
1203{
1204 struct fcnvme_ls_rjt *rjt = buf;
1205
1206 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1207 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1208 ls_cmd);
1209 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1210 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1211 rjt->rjt.reason_code = reason;
1212 rjt->rjt.reason_explanation = explanation;
1213 rjt->rjt.vendor = vendor;
1214
1215 return sizeof(struct fcnvme_ls_rjt);
1216}
1217
1218
1219enum {
1220 VERR_NO_ERROR = 0,
1221 VERR_CR_ASSOC_LEN = 1,
1222 VERR_CR_ASSOC_RQST_LEN = 2,
1223 VERR_CR_ASSOC_CMD = 3,
1224 VERR_CR_ASSOC_CMD_LEN = 4,
1225 VERR_ERSP_RATIO = 5,
1226 VERR_ASSOC_ALLOC_FAIL = 6,
1227 VERR_QUEUE_ALLOC_FAIL = 7,
1228 VERR_CR_CONN_LEN = 8,
1229 VERR_CR_CONN_RQST_LEN = 9,
1230 VERR_ASSOC_ID = 10,
1231 VERR_ASSOC_ID_LEN = 11,
1232 VERR_NO_ASSOC = 12,
1233 VERR_CONN_ID = 13,
1234 VERR_CONN_ID_LEN = 14,
1235 VERR_NO_CONN = 15,
1236 VERR_CR_CONN_CMD = 16,
1237 VERR_CR_CONN_CMD_LEN = 17,
1238 VERR_DISCONN_LEN = 18,
1239 VERR_DISCONN_RQST_LEN = 19,
1240 VERR_DISCONN_CMD = 20,
1241 VERR_DISCONN_CMD_LEN = 21,
1242 VERR_DISCONN_SCOPE = 22,
1243 VERR_RS_LEN = 23,
1244 VERR_RS_RQST_LEN = 24,
1245 VERR_RS_CMD = 25,
1246 VERR_RS_CMD_LEN = 26,
1247 VERR_RS_RCTL = 27,
1248 VERR_RS_RO = 28,
1249};
1250
1251static char *validation_errors[] = {
1252 "OK",
1253 "Bad CR_ASSOC Length",
1254 "Bad CR_ASSOC Rqst Length",
1255 "Not CR_ASSOC Cmd",
1256 "Bad CR_ASSOC Cmd Length",
1257 "Bad Ersp Ratio",
1258 "Association Allocation Failed",
1259 "Queue Allocation Failed",
1260 "Bad CR_CONN Length",
1261 "Bad CR_CONN Rqst Length",
1262 "Not Association ID",
1263 "Bad Association ID Length",
1264 "No Association",
1265 "Not Connection ID",
1266 "Bad Connection ID Length",
1267 "No Connection",
1268 "Not CR_CONN Cmd",
1269 "Bad CR_CONN Cmd Length",
1270 "Bad DISCONN Length",
1271 "Bad DISCONN Rqst Length",
1272 "Not DISCONN Cmd",
1273 "Bad DISCONN Cmd Length",
1274 "Bad Disconnect Scope",
1275 "Bad RS Length",
1276 "Bad RS Rqst Length",
1277 "Not RS Cmd",
1278 "Bad RS Cmd Length",
1279 "Bad RS R_CTL",
1280 "Bad RS Relative Offset",
1281};
1282
1283static void
1284nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1285 struct nvmet_fc_ls_iod *iod)
1286{
1287 struct fcnvme_ls_cr_assoc_rqst *rqst =
1288 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1289 struct fcnvme_ls_cr_assoc_acc *acc =
1290 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1291 struct nvmet_fc_tgt_queue *queue;
1292 int ret = 0;
1293
1294 memset(acc, 0, sizeof(*acc));
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1305 ret = VERR_CR_ASSOC_LEN;
1306 else if (be32_to_cpu(rqst->desc_list_len) <
1307 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1308 ret = VERR_CR_ASSOC_RQST_LEN;
1309 else if (rqst->assoc_cmd.desc_tag !=
1310 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1311 ret = VERR_CR_ASSOC_CMD;
1312 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1313 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1314 ret = VERR_CR_ASSOC_CMD_LEN;
1315 else if (!rqst->assoc_cmd.ersp_ratio ||
1316 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1317 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1318 ret = VERR_ERSP_RATIO;
1319
1320 else {
1321
1322 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1323 if (!iod->assoc)
1324 ret = VERR_ASSOC_ALLOC_FAIL;
1325 else {
1326 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1327 be16_to_cpu(rqst->assoc_cmd.sqsize));
1328 if (!queue)
1329 ret = VERR_QUEUE_ALLOC_FAIL;
1330 }
1331 }
1332
1333 if (ret) {
1334 dev_err(tgtport->dev,
1335 "Create Association LS failed: %s\n",
1336 validation_errors[ret]);
1337 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1338 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1339 FCNVME_RJT_RC_LOGIC,
1340 FCNVME_RJT_EXP_NONE, 0);
1341 return;
1342 }
1343
1344 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1345 atomic_set(&queue->connected, 1);
1346 queue->sqhd = 0;
1347
1348
1349
1350 iod->lsreq->rsplen = sizeof(*acc);
1351
1352 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1353 fcnvme_lsdesc_len(
1354 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1355 FCNVME_LS_CREATE_ASSOCIATION);
1356 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1357 acc->associd.desc_len =
1358 fcnvme_lsdesc_len(
1359 sizeof(struct fcnvme_lsdesc_assoc_id));
1360 acc->associd.association_id =
1361 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1362 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1363 acc->connectid.desc_len =
1364 fcnvme_lsdesc_len(
1365 sizeof(struct fcnvme_lsdesc_conn_id));
1366 acc->connectid.connection_id = acc->associd.association_id;
1367}
1368
1369static void
1370nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1371 struct nvmet_fc_ls_iod *iod)
1372{
1373 struct fcnvme_ls_cr_conn_rqst *rqst =
1374 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1375 struct fcnvme_ls_cr_conn_acc *acc =
1376 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1377 struct nvmet_fc_tgt_queue *queue;
1378 int ret = 0;
1379
1380 memset(acc, 0, sizeof(*acc));
1381
1382 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1383 ret = VERR_CR_CONN_LEN;
1384 else if (rqst->desc_list_len !=
1385 fcnvme_lsdesc_len(
1386 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1387 ret = VERR_CR_CONN_RQST_LEN;
1388 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1389 ret = VERR_ASSOC_ID;
1390 else if (rqst->associd.desc_len !=
1391 fcnvme_lsdesc_len(
1392 sizeof(struct fcnvme_lsdesc_assoc_id)))
1393 ret = VERR_ASSOC_ID_LEN;
1394 else if (rqst->connect_cmd.desc_tag !=
1395 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1396 ret = VERR_CR_CONN_CMD;
1397 else if (rqst->connect_cmd.desc_len !=
1398 fcnvme_lsdesc_len(
1399 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1400 ret = VERR_CR_CONN_CMD_LEN;
1401 else if (!rqst->connect_cmd.ersp_ratio ||
1402 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1403 be16_to_cpu(rqst->connect_cmd.sqsize)))
1404 ret = VERR_ERSP_RATIO;
1405
1406 else {
1407
1408 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1409 be64_to_cpu(rqst->associd.association_id));
1410 if (!iod->assoc)
1411 ret = VERR_NO_ASSOC;
1412 else {
1413 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1414 be16_to_cpu(rqst->connect_cmd.qid),
1415 be16_to_cpu(rqst->connect_cmd.sqsize));
1416 if (!queue)
1417 ret = VERR_QUEUE_ALLOC_FAIL;
1418
1419
1420 nvmet_fc_tgt_a_put(iod->assoc);
1421 }
1422 }
1423
1424 if (ret) {
1425 dev_err(tgtport->dev,
1426 "Create Connection LS failed: %s\n",
1427 validation_errors[ret]);
1428 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1429 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1430 (ret == VERR_NO_ASSOC) ?
1431 FCNVME_RJT_RC_INV_ASSOC :
1432 FCNVME_RJT_RC_LOGIC,
1433 FCNVME_RJT_EXP_NONE, 0);
1434 return;
1435 }
1436
1437 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1438 atomic_set(&queue->connected, 1);
1439 queue->sqhd = 0;
1440
1441
1442
1443 iod->lsreq->rsplen = sizeof(*acc);
1444
1445 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1446 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1447 FCNVME_LS_CREATE_CONNECTION);
1448 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1449 acc->connectid.desc_len =
1450 fcnvme_lsdesc_len(
1451 sizeof(struct fcnvme_lsdesc_conn_id));
1452 acc->connectid.connection_id =
1453 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1454 be16_to_cpu(rqst->connect_cmd.qid)));
1455}
1456
1457static void
1458nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1459 struct nvmet_fc_ls_iod *iod)
1460{
1461 struct fcnvme_ls_disconnect_rqst *rqst =
1462 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1463 struct fcnvme_ls_disconnect_acc *acc =
1464 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1465 struct nvmet_fc_tgt_queue *queue = NULL;
1466 struct nvmet_fc_tgt_assoc *assoc;
1467 int ret = 0;
1468 bool del_assoc = false;
1469
1470 memset(acc, 0, sizeof(*acc));
1471
1472 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1473 ret = VERR_DISCONN_LEN;
1474 else if (rqst->desc_list_len !=
1475 fcnvme_lsdesc_len(
1476 sizeof(struct fcnvme_ls_disconnect_rqst)))
1477 ret = VERR_DISCONN_RQST_LEN;
1478 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1479 ret = VERR_ASSOC_ID;
1480 else if (rqst->associd.desc_len !=
1481 fcnvme_lsdesc_len(
1482 sizeof(struct fcnvme_lsdesc_assoc_id)))
1483 ret = VERR_ASSOC_ID_LEN;
1484 else if (rqst->discon_cmd.desc_tag !=
1485 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1486 ret = VERR_DISCONN_CMD;
1487 else if (rqst->discon_cmd.desc_len !=
1488 fcnvme_lsdesc_len(
1489 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1490 ret = VERR_DISCONN_CMD_LEN;
1491 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1492 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1493 ret = VERR_DISCONN_SCOPE;
1494 else {
1495
1496 assoc = nvmet_fc_find_target_assoc(tgtport,
1497 be64_to_cpu(rqst->associd.association_id));
1498 iod->assoc = assoc;
1499 if (assoc) {
1500 if (rqst->discon_cmd.scope ==
1501 FCNVME_DISCONN_CONNECTION) {
1502 queue = nvmet_fc_find_target_queue(tgtport,
1503 be64_to_cpu(
1504 rqst->discon_cmd.id));
1505 if (!queue) {
1506 nvmet_fc_tgt_a_put(assoc);
1507 ret = VERR_NO_CONN;
1508 }
1509 }
1510 } else
1511 ret = VERR_NO_ASSOC;
1512 }
1513
1514 if (ret) {
1515 dev_err(tgtport->dev,
1516 "Disconnect LS failed: %s\n",
1517 validation_errors[ret]);
1518 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1519 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1520 (ret == VERR_NO_ASSOC) ?
1521 FCNVME_RJT_RC_INV_ASSOC :
1522 (ret == VERR_NO_CONN) ?
1523 FCNVME_RJT_RC_INV_CONN :
1524 FCNVME_RJT_RC_LOGIC,
1525 FCNVME_RJT_EXP_NONE, 0);
1526 return;
1527 }
1528
1529
1530
1531 iod->lsreq->rsplen = sizeof(*acc);
1532
1533 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1534 fcnvme_lsdesc_len(
1535 sizeof(struct fcnvme_ls_disconnect_acc)),
1536 FCNVME_LS_DISCONNECT);
1537
1538
1539
1540 if (queue) {
1541 int qid = queue->qid;
1542
1543 nvmet_fc_delete_target_queue(queue);
1544
1545
1546 nvmet_fc_tgt_q_put(queue);
1547
1548
1549 if (!qid)
1550 del_assoc = true;
1551 }
1552
1553
1554 nvmet_fc_tgt_a_put(iod->assoc);
1555
1556 if (del_assoc)
1557 nvmet_fc_delete_target_assoc(iod->assoc);
1558}
1559
1560
1561
1562
1563
1564static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1565
1566static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1567
1568static void
1569nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1570{
1571 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1572 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1573
1574 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1575 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1576 nvmet_fc_free_ls_iod(tgtport, iod);
1577 nvmet_fc_tgtport_put(tgtport);
1578}
1579
1580static void
1581nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1582 struct nvmet_fc_ls_iod *iod)
1583{
1584 int ret;
1585
1586 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1587 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1588
1589 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1590 if (ret)
1591 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1592}
1593
1594
1595
1596
1597static void
1598nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1599 struct nvmet_fc_ls_iod *iod)
1600{
1601 struct fcnvme_ls_rqst_w0 *w0 =
1602 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1603
1604 iod->lsreq->nvmet_fc_private = iod;
1605 iod->lsreq->rspbuf = iod->rspbuf;
1606 iod->lsreq->rspdma = iod->rspdma;
1607 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1608
1609 iod->lsreq->rsplen = 0;
1610
1611 iod->assoc = NULL;
1612
1613
1614
1615
1616
1617
1618 switch (w0->ls_cmd) {
1619 case FCNVME_LS_CREATE_ASSOCIATION:
1620
1621 nvmet_fc_ls_create_association(tgtport, iod);
1622 break;
1623 case FCNVME_LS_CREATE_CONNECTION:
1624
1625 nvmet_fc_ls_create_connection(tgtport, iod);
1626 break;
1627 case FCNVME_LS_DISCONNECT:
1628
1629 nvmet_fc_ls_disconnect(tgtport, iod);
1630 break;
1631 default:
1632 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1633 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1634 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1635 }
1636
1637 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1638}
1639
1640
1641
1642
1643static void
1644nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1645{
1646 struct nvmet_fc_ls_iod *iod =
1647 container_of(work, struct nvmet_fc_ls_iod, work);
1648 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1649
1650 nvmet_fc_handle_ls_rqst(tgtport, iod);
1651}
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671int
1672nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1673 struct nvmefc_tgt_ls_req *lsreq,
1674 void *lsreqbuf, u32 lsreqbuf_len)
1675{
1676 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1677 struct nvmet_fc_ls_iod *iod;
1678
1679 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1680 return -E2BIG;
1681
1682 if (!nvmet_fc_tgtport_get(tgtport))
1683 return -ESHUTDOWN;
1684
1685 iod = nvmet_fc_alloc_ls_iod(tgtport);
1686 if (!iod) {
1687 nvmet_fc_tgtport_put(tgtport);
1688 return -ENOENT;
1689 }
1690
1691 iod->lsreq = lsreq;
1692 iod->fcpreq = NULL;
1693 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1694 iod->rqstdatalen = lsreqbuf_len;
1695
1696 schedule_work(&iod->work);
1697
1698 return 0;
1699}
1700EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1701
1702
1703
1704
1705
1706
1707
1708
1709static int
1710nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1711{
1712 struct scatterlist *sg;
1713 unsigned int nent;
1714
1715 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
1716 if (!sg)
1717 goto out;
1718
1719 fod->data_sg = sg;
1720 fod->data_sg_cnt = nent;
1721 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1722 ((fod->io_dir == NVMET_FCP_WRITE) ?
1723 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1724
1725
1726 return 0;
1727
1728out:
1729 return NVME_SC_INTERNAL;
1730}
1731
1732static void
1733nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1734{
1735 if (!fod->data_sg || !fod->data_sg_cnt)
1736 return;
1737
1738 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1739 ((fod->io_dir == NVMET_FCP_WRITE) ?
1740 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1741 sgl_free(fod->data_sg);
1742 fod->data_sg = NULL;
1743 fod->data_sg_cnt = 0;
1744}
1745
1746
1747static bool
1748queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1749{
1750 u32 sqtail, used;
1751
1752
1753 sqtail = atomic_read(&q->sqtail) % q->sqsize;
1754
1755 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1756 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1757}
1758
1759
1760
1761
1762
1763static void
1764nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1765 struct nvmet_fc_fcp_iod *fod)
1766{
1767 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1768 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1769 struct nvme_completion *cqe = &ersp->cqe;
1770 u32 *cqewd = (u32 *)cqe;
1771 bool send_ersp = false;
1772 u32 rsn, rspcnt, xfr_length;
1773
1774 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1775 xfr_length = fod->req.transfer_len;
1776 else
1777 xfr_length = fod->offset;
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1799 if (!(rspcnt % fod->queue->ersp_ratio) ||
1800 sqe->opcode == nvme_fabrics_command ||
1801 xfr_length != fod->req.transfer_len ||
1802 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1803 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1804 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1805 send_ersp = true;
1806
1807
1808 fod->fcpreq->rspaddr = ersp;
1809 fod->fcpreq->rspdma = fod->rspdma;
1810
1811 if (!send_ersp) {
1812 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1813 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1814 } else {
1815 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1816 rsn = atomic_inc_return(&fod->queue->rsn);
1817 ersp->rsn = cpu_to_be32(rsn);
1818 ersp->xfrd_len = cpu_to_be32(xfr_length);
1819 fod->fcpreq->rsplen = sizeof(*ersp);
1820 }
1821
1822 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1823 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1824}
1825
1826static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1827
1828static void
1829nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1830 struct nvmet_fc_fcp_iod *fod)
1831{
1832 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1833
1834
1835 nvmet_fc_free_tgt_pgs(fod);
1836
1837
1838
1839
1840
1841
1842 if (!fod->aborted)
1843 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1844
1845 nvmet_fc_free_fcp_iod(fod->queue, fod);
1846}
1847
1848static void
1849nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1850 struct nvmet_fc_fcp_iod *fod)
1851{
1852 int ret;
1853
1854 fod->fcpreq->op = NVMET_FCOP_RSP;
1855 fod->fcpreq->timeout = 0;
1856
1857 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1858
1859 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1860 if (ret)
1861 nvmet_fc_abort_op(tgtport, fod);
1862}
1863
1864static void
1865nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1866 struct nvmet_fc_fcp_iod *fod, u8 op)
1867{
1868 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1869 unsigned long flags;
1870 u32 tlen;
1871 int ret;
1872
1873 fcpreq->op = op;
1874 fcpreq->offset = fod->offset;
1875 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1876
1877 tlen = min_t(u32, tgtport->max_sg_cnt * PAGE_SIZE,
1878 (fod->req.transfer_len - fod->offset));
1879 fcpreq->transfer_length = tlen;
1880 fcpreq->transferred_length = 0;
1881 fcpreq->fcp_error = 0;
1882 fcpreq->rsplen = 0;
1883
1884 fcpreq->sg = &fod->data_sg[fod->offset / PAGE_SIZE];
1885 fcpreq->sg_cnt = DIV_ROUND_UP(tlen, PAGE_SIZE);
1886
1887
1888
1889
1890
1891 if ((op == NVMET_FCOP_READDATA) &&
1892 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
1893 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1894 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1895 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1896 }
1897
1898 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1899 if (ret) {
1900
1901
1902
1903
1904
1905 fod->abort = true;
1906
1907 if (op == NVMET_FCOP_WRITEDATA) {
1908 spin_lock_irqsave(&fod->flock, flags);
1909 fod->writedataactive = false;
1910 spin_unlock_irqrestore(&fod->flock, flags);
1911 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1912 } else {
1913 fcpreq->fcp_error = ret;
1914 fcpreq->transferred_length = 0;
1915 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1916 }
1917 }
1918}
1919
1920static inline bool
1921__nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
1922{
1923 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1924 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1925
1926
1927 if (abort) {
1928 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
1929 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1930 return true;
1931 }
1932
1933 nvmet_fc_abort_op(tgtport, fod);
1934 return true;
1935 }
1936
1937 return false;
1938}
1939
1940
1941
1942
1943static void
1944nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
1945{
1946 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1947 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1948 unsigned long flags;
1949 bool abort;
1950
1951 spin_lock_irqsave(&fod->flock, flags);
1952 abort = fod->abort;
1953 fod->writedataactive = false;
1954 spin_unlock_irqrestore(&fod->flock, flags);
1955
1956 switch (fcpreq->op) {
1957
1958 case NVMET_FCOP_WRITEDATA:
1959 if (__nvmet_fc_fod_op_abort(fod, abort))
1960 return;
1961 if (fcpreq->fcp_error ||
1962 fcpreq->transferred_length != fcpreq->transfer_length) {
1963 spin_lock(&fod->flock);
1964 fod->abort = true;
1965 spin_unlock(&fod->flock);
1966
1967 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1968 return;
1969 }
1970
1971 fod->offset += fcpreq->transferred_length;
1972 if (fod->offset != fod->req.transfer_len) {
1973 spin_lock_irqsave(&fod->flock, flags);
1974 fod->writedataactive = true;
1975 spin_unlock_irqrestore(&fod->flock, flags);
1976
1977
1978 nvmet_fc_transfer_fcp_data(tgtport, fod,
1979 NVMET_FCOP_WRITEDATA);
1980 return;
1981 }
1982
1983
1984 nvmet_req_execute(&fod->req);
1985 break;
1986
1987 case NVMET_FCOP_READDATA:
1988 case NVMET_FCOP_READDATA_RSP:
1989 if (__nvmet_fc_fod_op_abort(fod, abort))
1990 return;
1991 if (fcpreq->fcp_error ||
1992 fcpreq->transferred_length != fcpreq->transfer_length) {
1993 nvmet_fc_abort_op(tgtport, fod);
1994 return;
1995 }
1996
1997
1998
1999 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2000
2001 nvmet_fc_free_tgt_pgs(fod);
2002 nvmet_fc_free_fcp_iod(fod->queue, fod);
2003 return;
2004 }
2005
2006 fod->offset += fcpreq->transferred_length;
2007 if (fod->offset != fod->req.transfer_len) {
2008
2009 nvmet_fc_transfer_fcp_data(tgtport, fod,
2010 NVMET_FCOP_READDATA);
2011 return;
2012 }
2013
2014
2015
2016
2017 nvmet_fc_free_tgt_pgs(fod);
2018
2019 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2020
2021 break;
2022
2023 case NVMET_FCOP_RSP:
2024 if (__nvmet_fc_fod_op_abort(fod, abort))
2025 return;
2026 nvmet_fc_free_fcp_iod(fod->queue, fod);
2027 break;
2028
2029 default:
2030 break;
2031 }
2032}
2033
2034static void
2035nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work)
2036{
2037 struct nvmet_fc_fcp_iod *fod =
2038 container_of(work, struct nvmet_fc_fcp_iod, done_work);
2039
2040 nvmet_fc_fod_op_done(fod);
2041}
2042
2043static void
2044nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2045{
2046 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2047 struct nvmet_fc_tgt_queue *queue = fod->queue;
2048
2049 if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR)
2050
2051 queue_work_on(queue->cpu, queue->work_q, &fod->done_work);
2052 else
2053 nvmet_fc_fod_op_done(fod);
2054}
2055
2056
2057
2058
2059static void
2060__nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2061 struct nvmet_fc_fcp_iod *fod, int status)
2062{
2063 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2064 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2065 unsigned long flags;
2066 bool abort;
2067
2068 spin_lock_irqsave(&fod->flock, flags);
2069 abort = fod->abort;
2070 spin_unlock_irqrestore(&fod->flock, flags);
2071
2072
2073 if (!status)
2074 fod->queue->sqhd = cqe->sq_head;
2075
2076 if (abort) {
2077 nvmet_fc_abort_op(tgtport, fod);
2078 return;
2079 }
2080
2081
2082 if (status) {
2083
2084 memset(cqe, 0, sizeof(*cqe));
2085 cqe->sq_head = fod->queue->sqhd;
2086 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2087 cqe->command_id = sqe->command_id;
2088 cqe->status = cpu_to_le16(status);
2089 } else {
2090
2091
2092
2093
2094
2095
2096 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2097
2098 nvmet_fc_transfer_fcp_data(tgtport, fod,
2099 NVMET_FCOP_READDATA);
2100 return;
2101 }
2102
2103
2104 }
2105
2106
2107 nvmet_fc_free_tgt_pgs(fod);
2108
2109 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2110}
2111
2112
2113static void
2114nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2115{
2116 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2117 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2118
2119 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2120}
2121
2122
2123
2124
2125
2126static void
2127nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2128 struct nvmet_fc_fcp_iod *fod)
2129{
2130 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2131 u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2132 int ret;
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2144
2145 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2146 fod->io_dir = NVMET_FCP_WRITE;
2147 if (!nvme_is_write(&cmdiu->sqe))
2148 goto transport_error;
2149 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2150 fod->io_dir = NVMET_FCP_READ;
2151 if (nvme_is_write(&cmdiu->sqe))
2152 goto transport_error;
2153 } else {
2154 fod->io_dir = NVMET_FCP_NODATA;
2155 if (xfrlen)
2156 goto transport_error;
2157 }
2158
2159 fod->req.cmd = &fod->cmdiubuf.sqe;
2160 fod->req.rsp = &fod->rspiubuf.cqe;
2161 fod->req.port = fod->queue->port;
2162
2163
2164 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2165
2166 fod->data_sg = NULL;
2167 fod->data_sg_cnt = 0;
2168
2169 ret = nvmet_req_init(&fod->req,
2170 &fod->queue->nvme_cq,
2171 &fod->queue->nvme_sq,
2172 &nvmet_fc_tgt_fcp_ops);
2173 if (!ret) {
2174
2175
2176 return;
2177 }
2178
2179 fod->req.transfer_len = xfrlen;
2180
2181
2182 atomic_inc(&fod->queue->sqtail);
2183
2184 if (fod->req.transfer_len) {
2185 ret = nvmet_fc_alloc_tgt_pgs(fod);
2186 if (ret) {
2187 nvmet_req_complete(&fod->req, ret);
2188 return;
2189 }
2190 }
2191 fod->req.sg = fod->data_sg;
2192 fod->req.sg_cnt = fod->data_sg_cnt;
2193 fod->offset = 0;
2194
2195 if (fod->io_dir == NVMET_FCP_WRITE) {
2196
2197 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2198 return;
2199 }
2200
2201
2202
2203
2204
2205
2206
2207 nvmet_req_execute(&fod->req);
2208 return;
2209
2210transport_error:
2211 nvmet_fc_abort_op(tgtport, fod);
2212}
2213
2214
2215
2216
2217static void
2218nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
2219{
2220 struct nvmet_fc_fcp_iod *fod =
2221 container_of(work, struct nvmet_fc_fcp_iod, work);
2222 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2223
2224 nvmet_fc_handle_fcp_rqst(tgtport, fod);
2225}
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274int
2275nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2276 struct nvmefc_tgt_fcp_req *fcpreq,
2277 void *cmdiubuf, u32 cmdiubuf_len)
2278{
2279 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2280 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2281 struct nvmet_fc_tgt_queue *queue;
2282 struct nvmet_fc_fcp_iod *fod;
2283 struct nvmet_fc_defer_fcp_req *deferfcp;
2284 unsigned long flags;
2285
2286
2287 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2288 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2289 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2290 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2291 return -EIO;
2292
2293 queue = nvmet_fc_find_target_queue(tgtport,
2294 be64_to_cpu(cmdiu->connection_id));
2295 if (!queue)
2296 return -ENOTCONN;
2297
2298
2299
2300
2301
2302
2303
2304
2305 spin_lock_irqsave(&queue->qlock, flags);
2306
2307 fod = nvmet_fc_alloc_fcp_iod(queue);
2308 if (fod) {
2309 spin_unlock_irqrestore(&queue->qlock, flags);
2310
2311 fcpreq->nvmet_fc_private = fod;
2312 fod->fcpreq = fcpreq;
2313
2314 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2315
2316 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2317
2318 return 0;
2319 }
2320
2321 if (!tgtport->ops->defer_rcv) {
2322 spin_unlock_irqrestore(&queue->qlock, flags);
2323
2324 nvmet_fc_tgt_q_put(queue);
2325 return -ENOENT;
2326 }
2327
2328 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2329 struct nvmet_fc_defer_fcp_req, req_list);
2330 if (deferfcp) {
2331
2332 list_del(&deferfcp->req_list);
2333 } else {
2334 spin_unlock_irqrestore(&queue->qlock, flags);
2335
2336
2337 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2338 if (!deferfcp) {
2339
2340 nvmet_fc_tgt_q_put(queue);
2341 return -ENOMEM;
2342 }
2343 spin_lock_irqsave(&queue->qlock, flags);
2344 }
2345
2346
2347 fcpreq->rspaddr = cmdiubuf;
2348 fcpreq->rsplen = cmdiubuf_len;
2349 deferfcp->fcp_req = fcpreq;
2350
2351
2352 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2353
2354
2355
2356 spin_unlock_irqrestore(&queue->qlock, flags);
2357
2358 return -EOVERFLOW;
2359}
2360EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385void
2386nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2387 struct nvmefc_tgt_fcp_req *fcpreq)
2388{
2389 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2390 struct nvmet_fc_tgt_queue *queue;
2391 unsigned long flags;
2392
2393 if (!fod || fod->fcpreq != fcpreq)
2394
2395 return;
2396
2397 queue = fod->queue;
2398
2399 spin_lock_irqsave(&queue->qlock, flags);
2400 if (fod->active) {
2401
2402
2403
2404
2405
2406 spin_lock(&fod->flock);
2407 fod->abort = true;
2408 fod->aborted = true;
2409 spin_unlock(&fod->flock);
2410 }
2411 spin_unlock_irqrestore(&queue->qlock, flags);
2412}
2413EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2414
2415
2416struct nvmet_fc_traddr {
2417 u64 nn;
2418 u64 pn;
2419};
2420
2421static int
2422__nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2423{
2424 u64 token64;
2425
2426 if (match_u64(sstr, &token64))
2427 return -EINVAL;
2428 *val = token64;
2429
2430 return 0;
2431}
2432
2433
2434
2435
2436
2437
2438static int
2439nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2440{
2441 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2442 substring_t wwn = { name, &name[sizeof(name)-1] };
2443 int nnoffset, pnoffset;
2444
2445
2446 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2447 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2448 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2449 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2450 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2451 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2452 NVME_FC_TRADDR_OXNNLEN;
2453 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2454 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2455 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2456 "pn-", NVME_FC_TRADDR_NNLEN))) {
2457 nnoffset = NVME_FC_TRADDR_NNLEN;
2458 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2459 } else
2460 goto out_einval;
2461
2462 name[0] = '0';
2463 name[1] = 'x';
2464 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2465
2466 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2467 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2468 goto out_einval;
2469
2470 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2471 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2472 goto out_einval;
2473
2474 return 0;
2475
2476out_einval:
2477 pr_warn("%s: bad traddr string\n", __func__);
2478 return -EINVAL;
2479}
2480
2481static int
2482nvmet_fc_add_port(struct nvmet_port *port)
2483{
2484 struct nvmet_fc_tgtport *tgtport;
2485 struct nvmet_fc_traddr traddr = { 0L, 0L };
2486 unsigned long flags;
2487 int ret;
2488
2489
2490 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2491 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2492 return -EINVAL;
2493
2494
2495
2496 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2497 sizeof(port->disc_addr.traddr));
2498 if (ret)
2499 return ret;
2500
2501 ret = -ENXIO;
2502 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2503 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2504 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2505 (tgtport->fc_target_port.port_name == traddr.pn)) {
2506 tgtport->port = port;
2507 ret = 0;
2508 break;
2509 }
2510 }
2511 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2512 return ret;
2513}
2514
2515static void
2516nvmet_fc_remove_port(struct nvmet_port *port)
2517{
2518
2519}
2520
2521static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2522 .owner = THIS_MODULE,
2523 .type = NVMF_TRTYPE_FC,
2524 .msdbd = 1,
2525 .add_port = nvmet_fc_add_port,
2526 .remove_port = nvmet_fc_remove_port,
2527 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2528 .delete_ctrl = nvmet_fc_delete_ctrl,
2529};
2530
2531static int __init nvmet_fc_init_module(void)
2532{
2533 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2534}
2535
2536static void __exit nvmet_fc_exit_module(void)
2537{
2538
2539 if (!list_empty(&nvmet_fc_target_list))
2540 pr_warn("%s: targetport list not empty\n", __func__);
2541
2542 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2543
2544 ida_destroy(&nvmet_fc_tgtport_cnt);
2545}
2546
2547module_init(nvmet_fc_init_module);
2548module_exit(nvmet_fc_exit_module);
2549
2550MODULE_LICENSE("GPL v2");
2551