1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/ctype.h>
40#include <linux/kthread.h>
41#include <linux/string.h>
42#include <linux/delay.h>
43#include <linux/atomic.h>
44#include <linux/inet.h>
45#include <rdma/ib_cache.h>
46#include <scsi/scsi_proto.h>
47#include <scsi/scsi_tcq.h>
48#include <target/target_core_base.h>
49#include <target/target_core_fabric.h>
50#include "ib_srpt.h"
51
52
53#define DRV_NAME "ib_srpt"
54
55#define SRPT_ID_STRING "Linux SRP target"
56
57#undef pr_fmt
58#define pr_fmt(fmt) DRV_NAME " " fmt
59
60MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61MODULE_DESCRIPTION("SCSI RDMA Protocol target driver");
62MODULE_LICENSE("Dual BSD/GPL");
63
64
65
66
67
68static u64 srpt_service_guid;
69static DEFINE_SPINLOCK(srpt_dev_lock);
70static LIST_HEAD(srpt_dev_list);
71
72static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
73module_param(srp_max_req_size, int, 0444);
74MODULE_PARM_DESC(srp_max_req_size,
75 "Maximum size of SRP request messages in bytes.");
76
77static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
78module_param(srpt_srq_size, int, 0444);
79MODULE_PARM_DESC(srpt_srq_size,
80 "Shared receive queue (SRQ) size.");
81
82static int srpt_get_u64_x(char *buffer, const struct kernel_param *kp)
83{
84 return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
85}
86module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
87 0444);
88MODULE_PARM_DESC(srpt_service_guid,
89 "Using this value for ioc_guid, id_ext, and cm_listen_id instead of using the node_guid of the first HCA.");
90
91static struct ib_client srpt_client;
92
93static DEFINE_MUTEX(rdma_cm_mutex);
94
95static u16 rdma_cm_port;
96static struct rdma_cm_id *rdma_cm_id;
97static void srpt_release_cmd(struct se_cmd *se_cmd);
98static void srpt_free_ch(struct kref *kref);
99static int srpt_queue_status(struct se_cmd *cmd);
100static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
101static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
102static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
103
104
105
106
107
108static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
109{
110 unsigned long flags;
111 enum rdma_ch_state prev;
112 bool changed = false;
113
114 spin_lock_irqsave(&ch->spinlock, flags);
115 prev = ch->state;
116 if (new > prev) {
117 ch->state = new;
118 changed = true;
119 }
120 spin_unlock_irqrestore(&ch->spinlock, flags);
121
122 return changed;
123}
124
125
126
127
128
129
130
131
132
133
134
135static void srpt_event_handler(struct ib_event_handler *handler,
136 struct ib_event *event)
137{
138 struct srpt_device *sdev;
139 struct srpt_port *sport;
140 u8 port_num;
141
142 sdev = ib_get_client_data(event->device, &srpt_client);
143 if (!sdev || sdev->device != event->device)
144 return;
145
146 pr_debug("ASYNC event= %d on device= %s\n", event->event,
147 dev_name(&sdev->device->dev));
148
149 switch (event->event) {
150 case IB_EVENT_PORT_ERR:
151 port_num = event->element.port_num - 1;
152 if (port_num < sdev->device->phys_port_cnt) {
153 sport = &sdev->port[port_num];
154 sport->lid = 0;
155 sport->sm_lid = 0;
156 } else {
157 WARN(true, "event %d: port_num %d out of range 1..%d\n",
158 event->event, port_num + 1,
159 sdev->device->phys_port_cnt);
160 }
161 break;
162 case IB_EVENT_PORT_ACTIVE:
163 case IB_EVENT_LID_CHANGE:
164 case IB_EVENT_PKEY_CHANGE:
165 case IB_EVENT_SM_CHANGE:
166 case IB_EVENT_CLIENT_REREGISTER:
167 case IB_EVENT_GID_CHANGE:
168
169 port_num = event->element.port_num - 1;
170 if (port_num < sdev->device->phys_port_cnt) {
171 sport = &sdev->port[port_num];
172 if (!sport->lid && !sport->sm_lid)
173 schedule_work(&sport->work);
174 } else {
175 WARN(true, "event %d: port_num %d out of range 1..%d\n",
176 event->event, port_num + 1,
177 sdev->device->phys_port_cnt);
178 }
179 break;
180 default:
181 pr_err("received unrecognized IB event %d\n", event->event);
182 break;
183 }
184}
185
186
187
188
189
190
191static void srpt_srq_event(struct ib_event *event, void *ctx)
192{
193 pr_debug("SRQ event %d\n", event->event);
194}
195
196static const char *get_ch_state_name(enum rdma_ch_state s)
197{
198 switch (s) {
199 case CH_CONNECTING:
200 return "connecting";
201 case CH_LIVE:
202 return "live";
203 case CH_DISCONNECTING:
204 return "disconnecting";
205 case CH_DRAINING:
206 return "draining";
207 case CH_DISCONNECTED:
208 return "disconnected";
209 }
210 return "???";
211}
212
213
214
215
216
217
218static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
219{
220 pr_debug("QP event %d on ch=%p sess_name=%s state=%d\n",
221 event->event, ch, ch->sess_name, ch->state);
222
223 switch (event->event) {
224 case IB_EVENT_COMM_EST:
225 if (ch->using_rdma_cm)
226 rdma_notify(ch->rdma_cm.cm_id, event->event);
227 else
228 ib_cm_notify(ch->ib_cm.cm_id, event->event);
229 break;
230 case IB_EVENT_QP_LAST_WQE_REACHED:
231 pr_debug("%s-%d, state %s: received Last WQE event.\n",
232 ch->sess_name, ch->qp->qp_num,
233 get_ch_state_name(ch->state));
234 break;
235 default:
236 pr_err("received unrecognized IB QP event %d\n", event->event);
237 break;
238 }
239}
240
241
242
243
244
245
246
247
248
249
250static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
251{
252 u16 id;
253 u8 tmp;
254
255 id = (slot - 1) / 2;
256 if (slot & 0x1) {
257 tmp = c_list[id] & 0xf;
258 c_list[id] = (value << 4) | tmp;
259 } else {
260 tmp = c_list[id] & 0xf0;
261 c_list[id] = (value & 0xf) | tmp;
262 }
263}
264
265
266
267
268
269
270
271
272static void srpt_get_class_port_info(struct ib_dm_mad *mad)
273{
274 struct ib_class_port_info *cif;
275
276 cif = (struct ib_class_port_info *)mad->data;
277 memset(cif, 0, sizeof(*cif));
278 cif->base_version = 1;
279 cif->class_version = 1;
280
281 ib_set_cpi_resp_time(cif, 20);
282 mad->mad_hdr.status = 0;
283}
284
285
286
287
288
289
290
291
292static void srpt_get_iou(struct ib_dm_mad *mad)
293{
294 struct ib_dm_iou_info *ioui;
295 u8 slot;
296 int i;
297
298 ioui = (struct ib_dm_iou_info *)mad->data;
299 ioui->change_id = cpu_to_be16(1);
300 ioui->max_controllers = 16;
301
302
303 srpt_set_ioc(ioui->controller_list, 1, 1);
304 for (i = 1, slot = 2; i < 16; i++, slot++)
305 srpt_set_ioc(ioui->controller_list, slot, 0);
306
307 mad->mad_hdr.status = 0;
308}
309
310
311
312
313
314
315
316
317
318
319
320static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
321 struct ib_dm_mad *mad)
322{
323 struct srpt_device *sdev = sport->sdev;
324 struct ib_dm_ioc_profile *iocp;
325 int send_queue_depth;
326
327 iocp = (struct ib_dm_ioc_profile *)mad->data;
328
329 if (!slot || slot > 16) {
330 mad->mad_hdr.status
331 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
332 return;
333 }
334
335 if (slot > 2) {
336 mad->mad_hdr.status
337 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
338 return;
339 }
340
341 if (sdev->use_srq)
342 send_queue_depth = sdev->srq_size;
343 else
344 send_queue_depth = min(MAX_SRPT_RQ_SIZE,
345 sdev->device->attrs.max_qp_wr);
346
347 memset(iocp, 0, sizeof(*iocp));
348 strcpy(iocp->id_string, SRPT_ID_STRING);
349 iocp->guid = cpu_to_be64(srpt_service_guid);
350 iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
351 iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
352 iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
353 iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
354 iocp->subsys_device_id = 0x0;
355 iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
356 iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
357 iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
358 iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
359 iocp->send_queue_depth = cpu_to_be16(send_queue_depth);
360 iocp->rdma_read_depth = 4;
361 iocp->send_size = cpu_to_be32(srp_max_req_size);
362 iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
363 1U << 24));
364 iocp->num_svc_entries = 1;
365 iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
366 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
367
368 mad->mad_hdr.status = 0;
369}
370
371
372
373
374
375
376
377
378
379
380
381
382static void srpt_get_svc_entries(u64 ioc_guid,
383 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
384{
385 struct ib_dm_svc_entries *svc_entries;
386
387 WARN_ON(!ioc_guid);
388
389 if (!slot || slot > 16) {
390 mad->mad_hdr.status
391 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
392 return;
393 }
394
395 if (slot > 2 || lo > hi || hi > 1) {
396 mad->mad_hdr.status
397 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
398 return;
399 }
400
401 svc_entries = (struct ib_dm_svc_entries *)mad->data;
402 memset(svc_entries, 0, sizeof(*svc_entries));
403 svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
404 snprintf(svc_entries->service_entries[0].name,
405 sizeof(svc_entries->service_entries[0].name),
406 "%s%016llx",
407 SRP_SERVICE_NAME_PREFIX,
408 ioc_guid);
409
410 mad->mad_hdr.status = 0;
411}
412
413
414
415
416
417
418
419static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
420 struct ib_dm_mad *rsp_mad)
421{
422 u16 attr_id;
423 u32 slot;
424 u8 hi, lo;
425
426 attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
427 switch (attr_id) {
428 case DM_ATTR_CLASS_PORT_INFO:
429 srpt_get_class_port_info(rsp_mad);
430 break;
431 case DM_ATTR_IOU_INFO:
432 srpt_get_iou(rsp_mad);
433 break;
434 case DM_ATTR_IOC_PROFILE:
435 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
436 srpt_get_ioc(sp, slot, rsp_mad);
437 break;
438 case DM_ATTR_SVC_ENTRIES:
439 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
440 hi = (u8) ((slot >> 8) & 0xff);
441 lo = (u8) (slot & 0xff);
442 slot = (u16) ((slot >> 16) & 0xffff);
443 srpt_get_svc_entries(srpt_service_guid,
444 slot, hi, lo, rsp_mad);
445 break;
446 default:
447 rsp_mad->mad_hdr.status =
448 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
449 break;
450 }
451}
452
453
454
455
456
457
458static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
459 struct ib_mad_send_wc *mad_wc)
460{
461 rdma_destroy_ah(mad_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE);
462 ib_free_send_mad(mad_wc->send_buf);
463}
464
465
466
467
468
469
470
471static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
472 struct ib_mad_send_buf *send_buf,
473 struct ib_mad_recv_wc *mad_wc)
474{
475 struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
476 struct ib_ah *ah;
477 struct ib_mad_send_buf *rsp;
478 struct ib_dm_mad *dm_mad;
479
480 if (!mad_wc || !mad_wc->recv_buf.mad)
481 return;
482
483 ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
484 mad_wc->recv_buf.grh, mad_agent->port_num);
485 if (IS_ERR(ah))
486 goto err;
487
488 BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
489
490 rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
491 mad_wc->wc->pkey_index, 0,
492 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
493 GFP_KERNEL,
494 IB_MGMT_BASE_VERSION);
495 if (IS_ERR(rsp))
496 goto err_rsp;
497
498 rsp->ah = ah;
499
500 dm_mad = rsp->mad;
501 memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
502 dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
503 dm_mad->mad_hdr.status = 0;
504
505 switch (mad_wc->recv_buf.mad->mad_hdr.method) {
506 case IB_MGMT_METHOD_GET:
507 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
508 break;
509 case IB_MGMT_METHOD_SET:
510 dm_mad->mad_hdr.status =
511 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
512 break;
513 default:
514 dm_mad->mad_hdr.status =
515 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
516 break;
517 }
518
519 if (!ib_post_send_mad(rsp, NULL)) {
520 ib_free_recv_mad(mad_wc);
521
522 return;
523 }
524
525 ib_free_send_mad(rsp);
526
527err_rsp:
528 rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
529err:
530 ib_free_recv_mad(mad_wc);
531}
532
533static int srpt_format_guid(char *buf, unsigned int size, const __be64 *guid)
534{
535 const __be16 *g = (const __be16 *)guid;
536
537 return snprintf(buf, size, "%04x:%04x:%04x:%04x",
538 be16_to_cpu(g[0]), be16_to_cpu(g[1]),
539 be16_to_cpu(g[2]), be16_to_cpu(g[3]));
540}
541
542
543
544
545
546
547
548
549
550
551
552static int srpt_refresh_port(struct srpt_port *sport)
553{
554 struct ib_mad_reg_req reg_req;
555 struct ib_port_modify port_modify;
556 struct ib_port_attr port_attr;
557 int ret;
558
559 memset(&port_modify, 0, sizeof(port_modify));
560 port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
561 port_modify.clr_port_cap_mask = 0;
562
563 ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
564 if (ret)
565 goto err_mod_port;
566
567 ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
568 if (ret)
569 goto err_query_port;
570
571 sport->sm_lid = port_attr.sm_lid;
572 sport->lid = port_attr.lid;
573
574 ret = rdma_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
575 if (ret)
576 goto err_query_port;
577
578 sport->port_guid_wwn.priv = sport;
579 srpt_format_guid(sport->port_guid, sizeof(sport->port_guid),
580 &sport->gid.global.interface_id);
581 sport->port_gid_wwn.priv = sport;
582 snprintf(sport->port_gid, sizeof(sport->port_gid),
583 "0x%016llx%016llx",
584 be64_to_cpu(sport->gid.global.subnet_prefix),
585 be64_to_cpu(sport->gid.global.interface_id));
586
587 if (!sport->mad_agent) {
588 memset(®_req, 0, sizeof(reg_req));
589 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
590 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
591 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
592 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
593
594 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
595 sport->port,
596 IB_QPT_GSI,
597 ®_req, 0,
598 srpt_mad_send_handler,
599 srpt_mad_recv_handler,
600 sport, 0);
601 if (IS_ERR(sport->mad_agent)) {
602 ret = PTR_ERR(sport->mad_agent);
603 sport->mad_agent = NULL;
604 goto err_query_port;
605 }
606 }
607
608 return 0;
609
610err_query_port:
611
612 port_modify.set_port_cap_mask = 0;
613 port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
614 ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
615
616err_mod_port:
617
618 return ret;
619}
620
621
622
623
624
625
626
627static void srpt_unregister_mad_agent(struct srpt_device *sdev)
628{
629 struct ib_port_modify port_modify = {
630 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
631 };
632 struct srpt_port *sport;
633 int i;
634
635 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
636 sport = &sdev->port[i - 1];
637 WARN_ON(sport->port != i);
638 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
639 pr_err("disabling MAD processing failed.\n");
640 if (sport->mad_agent) {
641 ib_unregister_mad_agent(sport->mad_agent);
642 sport->mad_agent = NULL;
643 }
644 }
645}
646
647
648
649
650
651
652
653
654static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
655 int ioctx_size,
656 struct kmem_cache *buf_cache,
657 enum dma_data_direction dir)
658{
659 struct srpt_ioctx *ioctx;
660
661 ioctx = kzalloc(ioctx_size, GFP_KERNEL);
662 if (!ioctx)
663 goto err;
664
665 ioctx->buf = kmem_cache_alloc(buf_cache, GFP_KERNEL);
666 if (!ioctx->buf)
667 goto err_free_ioctx;
668
669 ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
670 kmem_cache_size(buf_cache), dir);
671 if (ib_dma_mapping_error(sdev->device, ioctx->dma))
672 goto err_free_buf;
673
674 return ioctx;
675
676err_free_buf:
677 kmem_cache_free(buf_cache, ioctx->buf);
678err_free_ioctx:
679 kfree(ioctx);
680err:
681 return NULL;
682}
683
684
685
686
687
688
689
690
691static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
692 struct kmem_cache *buf_cache,
693 enum dma_data_direction dir)
694{
695 if (!ioctx)
696 return;
697
698 ib_dma_unmap_single(sdev->device, ioctx->dma,
699 kmem_cache_size(buf_cache), dir);
700 kmem_cache_free(buf_cache, ioctx->buf);
701 kfree(ioctx);
702}
703
704
705
706
707
708
709
710
711
712
713
714static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
715 int ring_size, int ioctx_size,
716 struct kmem_cache *buf_cache,
717 int alignment_offset,
718 enum dma_data_direction dir)
719{
720 struct srpt_ioctx **ring;
721 int i;
722
723 WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx) &&
724 ioctx_size != sizeof(struct srpt_send_ioctx));
725
726 ring = kvmalloc_array(ring_size, sizeof(ring[0]), GFP_KERNEL);
727 if (!ring)
728 goto out;
729 for (i = 0; i < ring_size; ++i) {
730 ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, buf_cache, dir);
731 if (!ring[i])
732 goto err;
733 ring[i]->index = i;
734 ring[i]->offset = alignment_offset;
735 }
736 goto out;
737
738err:
739 while (--i >= 0)
740 srpt_free_ioctx(sdev, ring[i], buf_cache, dir);
741 kvfree(ring);
742 ring = NULL;
743out:
744 return ring;
745}
746
747
748
749
750
751
752
753
754
755static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
756 struct srpt_device *sdev, int ring_size,
757 struct kmem_cache *buf_cache,
758 enum dma_data_direction dir)
759{
760 int i;
761
762 if (!ioctx_ring)
763 return;
764
765 for (i = 0; i < ring_size; ++i)
766 srpt_free_ioctx(sdev, ioctx_ring[i], buf_cache, dir);
767 kvfree(ioctx_ring);
768}
769
770
771
772
773
774
775
776
777
778static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
779 enum srpt_command_state new)
780{
781 enum srpt_command_state previous;
782
783 previous = ioctx->state;
784 if (previous != SRPT_STATE_DONE)
785 ioctx->state = new;
786
787 return previous;
788}
789
790
791
792
793
794
795
796
797
798static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
799 enum srpt_command_state old,
800 enum srpt_command_state new)
801{
802 enum srpt_command_state previous;
803
804 WARN_ON(!ioctx);
805 WARN_ON(old == SRPT_STATE_DONE);
806 WARN_ON(new == SRPT_STATE_NEW);
807
808 previous = ioctx->state;
809 if (previous == old)
810 ioctx->state = new;
811
812 return previous == old;
813}
814
815
816
817
818
819
820
821static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch,
822 struct srpt_recv_ioctx *ioctx)
823{
824 struct ib_sge list;
825 struct ib_recv_wr wr;
826
827 BUG_ON(!sdev);
828 list.addr = ioctx->ioctx.dma + ioctx->ioctx.offset;
829 list.length = srp_max_req_size;
830 list.lkey = sdev->lkey;
831
832 ioctx->ioctx.cqe.done = srpt_recv_done;
833 wr.wr_cqe = &ioctx->ioctx.cqe;
834 wr.next = NULL;
835 wr.sg_list = &list;
836 wr.num_sge = 1;
837
838 if (sdev->use_srq)
839 return ib_post_srq_recv(sdev->srq, &wr, NULL);
840 else
841 return ib_post_recv(ch->qp, &wr, NULL);
842}
843
844
845
846
847
848
849
850
851
852
853static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
854{
855 struct ib_rdma_wr wr = {
856 .wr = {
857 .next = NULL,
858 { .wr_cqe = &ch->zw_cqe, },
859 .opcode = IB_WR_RDMA_WRITE,
860 .send_flags = IB_SEND_SIGNALED,
861 }
862 };
863
864 pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
865 ch->qp->qp_num);
866
867 return ib_post_send(ch->qp, &wr.wr, NULL);
868}
869
870static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
871{
872 struct srpt_rdma_ch *ch = cq->cq_context;
873
874 pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
875 wc->status);
876
877 if (wc->status == IB_WC_SUCCESS) {
878 srpt_process_wait_list(ch);
879 } else {
880 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
881 schedule_work(&ch->release_work);
882 else
883 pr_debug("%s-%d: already disconnected.\n",
884 ch->sess_name, ch->qp->qp_num);
885 }
886}
887
888static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
889 struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
890 unsigned *sg_cnt)
891{
892 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
893 struct srpt_rdma_ch *ch = ioctx->ch;
894 struct scatterlist *prev = NULL;
895 unsigned prev_nents;
896 int ret, i;
897
898 if (nbufs == 1) {
899 ioctx->rw_ctxs = &ioctx->s_rw_ctx;
900 } else {
901 ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
902 GFP_KERNEL);
903 if (!ioctx->rw_ctxs)
904 return -ENOMEM;
905 }
906
907 for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
908 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
909 u64 remote_addr = be64_to_cpu(db->va);
910 u32 size = be32_to_cpu(db->len);
911 u32 rkey = be32_to_cpu(db->key);
912
913 ret = target_alloc_sgl(&ctx->sg, &ctx->nents, size, false,
914 i < nbufs - 1);
915 if (ret)
916 goto unwind;
917
918 ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port,
919 ctx->sg, ctx->nents, 0, remote_addr, rkey, dir);
920 if (ret < 0) {
921 target_free_sgl(ctx->sg, ctx->nents);
922 goto unwind;
923 }
924
925 ioctx->n_rdma += ret;
926 ioctx->n_rw_ctx++;
927
928 if (prev) {
929 sg_unmark_end(&prev[prev_nents - 1]);
930 sg_chain(prev, prev_nents + 1, ctx->sg);
931 } else {
932 *sg = ctx->sg;
933 }
934
935 prev = ctx->sg;
936 prev_nents = ctx->nents;
937
938 *sg_cnt += ctx->nents;
939 }
940
941 return 0;
942
943unwind:
944 while (--i >= 0) {
945 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
946
947 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
948 ctx->sg, ctx->nents, dir);
949 target_free_sgl(ctx->sg, ctx->nents);
950 }
951 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
952 kfree(ioctx->rw_ctxs);
953 return ret;
954}
955
956static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
957 struct srpt_send_ioctx *ioctx)
958{
959 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
960 int i;
961
962 for (i = 0; i < ioctx->n_rw_ctx; i++) {
963 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
964
965 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
966 ctx->sg, ctx->nents, dir);
967 target_free_sgl(ctx->sg, ctx->nents);
968 }
969
970 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
971 kfree(ioctx->rw_ctxs);
972}
973
974static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
975{
976
977
978
979
980
981 BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
982 !__same_type(srp_cmd->add_data[0], (u8)0));
983
984
985
986
987
988
989 return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
990}
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx,
1012 struct srpt_send_ioctx *ioctx,
1013 struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
1014 struct scatterlist **sg, unsigned int *sg_cnt, u64 *data_len,
1015 u16 imm_data_offset)
1016{
1017 BUG_ON(!dir);
1018 BUG_ON(!data_len);
1019
1020
1021
1022
1023
1024
1025 if (srp_cmd->buf_fmt & 0xf)
1026
1027 *dir = DMA_FROM_DEVICE;
1028 else if (srp_cmd->buf_fmt >> 4)
1029
1030 *dir = DMA_TO_DEVICE;
1031 else
1032 *dir = DMA_NONE;
1033
1034
1035 ioctx->cmd.data_direction = *dir;
1036
1037 if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
1038 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
1039 struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
1040
1041 *data_len = be32_to_cpu(db->len);
1042 return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
1043 } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
1044 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
1045 struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
1046 int nbufs = be32_to_cpu(idb->table_desc.len) /
1047 sizeof(struct srp_direct_buf);
1048
1049 if (nbufs >
1050 (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
1051 pr_err("received unsupported SRP_CMD request type (%u out + %u in != %u / %zu)\n",
1052 srp_cmd->data_out_desc_cnt,
1053 srp_cmd->data_in_desc_cnt,
1054 be32_to_cpu(idb->table_desc.len),
1055 sizeof(struct srp_direct_buf));
1056 return -EINVAL;
1057 }
1058
1059 *data_len = be32_to_cpu(idb->len);
1060 return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
1061 sg, sg_cnt);
1062 } else if ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_IMM) {
1063 struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd);
1064 void *data = (void *)srp_cmd + imm_data_offset;
1065 uint32_t len = be32_to_cpu(imm_buf->len);
1066 uint32_t req_size = imm_data_offset + len;
1067
1068 if (req_size > srp_max_req_size) {
1069 pr_err("Immediate data (length %d + %d) exceeds request size %d\n",
1070 imm_data_offset, len, srp_max_req_size);
1071 return -EINVAL;
1072 }
1073 if (recv_ioctx->byte_len < req_size) {
1074 pr_err("Received too few data - %d < %d\n",
1075 recv_ioctx->byte_len, req_size);
1076 return -EIO;
1077 }
1078
1079
1080
1081
1082 if ((void *)(imm_buf + 1) > (void *)data) {
1083 pr_err("Received invalid write request\n");
1084 return -EINVAL;
1085 }
1086 *data_len = len;
1087 ioctx->recv_ioctx = recv_ioctx;
1088 if ((uintptr_t)data & 511) {
1089 pr_warn_once("Internal error - the receive buffers are not aligned properly.\n");
1090 return -EINVAL;
1091 }
1092 sg_init_one(&ioctx->imm_sg, data, len);
1093 *sg = &ioctx->imm_sg;
1094 *sg_cnt = 1;
1095 return 0;
1096 } else {
1097 *data_len = 0;
1098 return 0;
1099 }
1100}
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1111{
1112 struct ib_qp_attr *attr;
1113 int ret;
1114
1115 WARN_ON_ONCE(ch->using_rdma_cm);
1116
1117 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1118 if (!attr)
1119 return -ENOMEM;
1120
1121 attr->qp_state = IB_QPS_INIT;
1122 attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
1123 attr->port_num = ch->sport->port;
1124
1125 ret = ib_find_cached_pkey(ch->sport->sdev->device, ch->sport->port,
1126 ch->pkey, &attr->pkey_index);
1127 if (ret < 0)
1128 pr_err("Translating pkey %#x failed (%d) - using index 0\n",
1129 ch->pkey, ret);
1130
1131 ret = ib_modify_qp(qp, attr,
1132 IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
1133 IB_QP_PKEY_INDEX);
1134
1135 kfree(attr);
1136 return ret;
1137}
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1151{
1152 struct ib_qp_attr qp_attr;
1153 int attr_mask;
1154 int ret;
1155
1156 WARN_ON_ONCE(ch->using_rdma_cm);
1157
1158 qp_attr.qp_state = IB_QPS_RTR;
1159 ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask);
1160 if (ret)
1161 goto out;
1162
1163 qp_attr.max_dest_rd_atomic = 4;
1164
1165 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1166
1167out:
1168 return ret;
1169}
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1183{
1184 struct ib_qp_attr qp_attr;
1185 int attr_mask;
1186 int ret;
1187
1188 qp_attr.qp_state = IB_QPS_RTS;
1189 ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask);
1190 if (ret)
1191 goto out;
1192
1193 qp_attr.max_rd_atomic = 4;
1194
1195 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1196
1197out:
1198 return ret;
1199}
1200
1201
1202
1203
1204
1205static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1206{
1207 struct ib_qp_attr qp_attr;
1208
1209 qp_attr.qp_state = IB_QPS_ERR;
1210 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1211}
1212
1213
1214
1215
1216
1217static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1218{
1219 struct srpt_send_ioctx *ioctx;
1220 int tag, cpu;
1221
1222 BUG_ON(!ch);
1223
1224 tag = sbitmap_queue_get(&ch->sess->sess_tag_pool, &cpu);
1225 if (tag < 0)
1226 return NULL;
1227
1228 ioctx = ch->ioctx_ring[tag];
1229 BUG_ON(ioctx->ch != ch);
1230 ioctx->state = SRPT_STATE_NEW;
1231 WARN_ON_ONCE(ioctx->recv_ioctx);
1232 ioctx->n_rdma = 0;
1233 ioctx->n_rw_ctx = 0;
1234 ioctx->queue_status_only = false;
1235
1236
1237
1238
1239 memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1240 memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1241 ioctx->cmd.map_tag = tag;
1242 ioctx->cmd.map_cpu = cpu;
1243
1244 return ioctx;
1245}
1246
1247
1248
1249
1250
1251static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1252{
1253 enum srpt_command_state state;
1254
1255 BUG_ON(!ioctx);
1256
1257
1258
1259
1260
1261
1262 state = ioctx->state;
1263 switch (state) {
1264 case SRPT_STATE_NEED_DATA:
1265 ioctx->state = SRPT_STATE_DATA_IN;
1266 break;
1267 case SRPT_STATE_CMD_RSP_SENT:
1268 case SRPT_STATE_MGMT_RSP_SENT:
1269 ioctx->state = SRPT_STATE_DONE;
1270 break;
1271 default:
1272 WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1273 __func__, state);
1274 break;
1275 }
1276
1277 pr_debug("Aborting cmd with state %d -> %d and tag %lld\n", state,
1278 ioctx->state, ioctx->cmd.tag);
1279
1280 switch (state) {
1281 case SRPT_STATE_NEW:
1282 case SRPT_STATE_DATA_IN:
1283 case SRPT_STATE_MGMT:
1284 case SRPT_STATE_DONE:
1285
1286
1287
1288
1289 break;
1290 case SRPT_STATE_NEED_DATA:
1291 pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1292 transport_generic_request_failure(&ioctx->cmd,
1293 TCM_CHECK_CONDITION_ABORT_CMD);
1294 break;
1295 case SRPT_STATE_CMD_RSP_SENT:
1296
1297
1298
1299
1300 transport_generic_free_cmd(&ioctx->cmd, 0);
1301 break;
1302 case SRPT_STATE_MGMT_RSP_SENT:
1303 transport_generic_free_cmd(&ioctx->cmd, 0);
1304 break;
1305 default:
1306 WARN(1, "Unexpected command state (%d)", state);
1307 break;
1308 }
1309
1310 return state;
1311}
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1324{
1325 struct srpt_rdma_ch *ch = cq->cq_context;
1326 struct srpt_send_ioctx *ioctx =
1327 container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
1328
1329 WARN_ON(ioctx->n_rdma <= 0);
1330 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1331 ioctx->n_rdma = 0;
1332
1333 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1334 pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1335 ioctx, wc->status);
1336 srpt_abort_cmd(ioctx);
1337 return;
1338 }
1339
1340 if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1341 SRPT_STATE_DATA_IN))
1342 target_execute_cmd(&ioctx->cmd);
1343 else
1344 pr_err("%s[%d]: wrong state = %d\n", __func__,
1345 __LINE__, ioctx->state);
1346}
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1364 struct srpt_send_ioctx *ioctx, u64 tag,
1365 int status)
1366{
1367 struct srp_rsp *srp_rsp;
1368 const u8 *sense_data;
1369 int sense_data_len, max_sense_len;
1370
1371
1372
1373
1374
1375 WARN_ON(status & 1);
1376
1377 srp_rsp = ioctx->ioctx.buf;
1378 BUG_ON(!srp_rsp);
1379
1380 sense_data = ioctx->sense_data;
1381 sense_data_len = ioctx->cmd.scsi_sense_length;
1382 WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1383
1384 memset(srp_rsp, 0, sizeof(*srp_rsp));
1385 srp_rsp->opcode = SRP_RSP;
1386 srp_rsp->req_lim_delta =
1387 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1388 srp_rsp->tag = tag;
1389 srp_rsp->status = status;
1390
1391 if (sense_data_len) {
1392 BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1393 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1394 if (sense_data_len > max_sense_len) {
1395 pr_warn("truncated sense data from %d to %d bytes\n",
1396 sense_data_len, max_sense_len);
1397 sense_data_len = max_sense_len;
1398 }
1399
1400 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1401 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1402 memcpy(srp_rsp + 1, sense_data, sense_data_len);
1403 }
1404
1405 return sizeof(*srp_rsp) + sense_data_len;
1406}
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1422 struct srpt_send_ioctx *ioctx,
1423 u8 rsp_code, u64 tag)
1424{
1425 struct srp_rsp *srp_rsp;
1426 int resp_data_len;
1427 int resp_len;
1428
1429 resp_data_len = 4;
1430 resp_len = sizeof(*srp_rsp) + resp_data_len;
1431
1432 srp_rsp = ioctx->ioctx.buf;
1433 BUG_ON(!srp_rsp);
1434 memset(srp_rsp, 0, sizeof(*srp_rsp));
1435
1436 srp_rsp->opcode = SRP_RSP;
1437 srp_rsp->req_lim_delta =
1438 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1439 srp_rsp->tag = tag;
1440
1441 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1442 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1443 srp_rsp->data[3] = rsp_code;
1444
1445 return resp_len;
1446}
1447
1448static int srpt_check_stop_free(struct se_cmd *cmd)
1449{
1450 struct srpt_send_ioctx *ioctx = container_of(cmd,
1451 struct srpt_send_ioctx, cmd);
1452
1453 return target_put_sess_cmd(&ioctx->cmd);
1454}
1455
1456
1457
1458
1459
1460
1461
1462static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1463 struct srpt_recv_ioctx *recv_ioctx,
1464 struct srpt_send_ioctx *send_ioctx)
1465{
1466 struct se_cmd *cmd;
1467 struct srp_cmd *srp_cmd;
1468 struct scatterlist *sg = NULL;
1469 unsigned sg_cnt = 0;
1470 u64 data_len;
1471 enum dma_data_direction dir;
1472 int rc;
1473
1474 BUG_ON(!send_ioctx);
1475
1476 srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1477 cmd = &send_ioctx->cmd;
1478 cmd->tag = srp_cmd->tag;
1479
1480 switch (srp_cmd->task_attr) {
1481 case SRP_CMD_SIMPLE_Q:
1482 cmd->sam_task_attr = TCM_SIMPLE_TAG;
1483 break;
1484 case SRP_CMD_ORDERED_Q:
1485 default:
1486 cmd->sam_task_attr = TCM_ORDERED_TAG;
1487 break;
1488 case SRP_CMD_HEAD_OF_Q:
1489 cmd->sam_task_attr = TCM_HEAD_TAG;
1490 break;
1491 case SRP_CMD_ACA:
1492 cmd->sam_task_attr = TCM_ACA_TAG;
1493 break;
1494 }
1495
1496 rc = srpt_get_desc_tbl(recv_ioctx, send_ioctx, srp_cmd, &dir,
1497 &sg, &sg_cnt, &data_len, ch->imm_data_offset);
1498 if (rc) {
1499 if (rc != -EAGAIN) {
1500 pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1501 srp_cmd->tag);
1502 }
1503 goto busy;
1504 }
1505
1506 rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
1507 &send_ioctx->sense_data[0],
1508 scsilun_to_int(&srp_cmd->lun), data_len,
1509 TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF,
1510 sg, sg_cnt, NULL, 0, NULL, 0);
1511 if (rc != 0) {
1512 pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1513 srp_cmd->tag);
1514 goto busy;
1515 }
1516 return;
1517
1518busy:
1519 target_send_busy(cmd);
1520}
1521
1522static int srp_tmr_to_tcm(int fn)
1523{
1524 switch (fn) {
1525 case SRP_TSK_ABORT_TASK:
1526 return TMR_ABORT_TASK;
1527 case SRP_TSK_ABORT_TASK_SET:
1528 return TMR_ABORT_TASK_SET;
1529 case SRP_TSK_CLEAR_TASK_SET:
1530 return TMR_CLEAR_TASK_SET;
1531 case SRP_TSK_LUN_RESET:
1532 return TMR_LUN_RESET;
1533 case SRP_TSK_CLEAR_ACA:
1534 return TMR_CLEAR_ACA;
1535 default:
1536 return -1;
1537 }
1538}
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1552 struct srpt_recv_ioctx *recv_ioctx,
1553 struct srpt_send_ioctx *send_ioctx)
1554{
1555 struct srp_tsk_mgmt *srp_tsk;
1556 struct se_cmd *cmd;
1557 struct se_session *sess = ch->sess;
1558 int tcm_tmr;
1559 int rc;
1560
1561 BUG_ON(!send_ioctx);
1562
1563 srp_tsk = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1564 cmd = &send_ioctx->cmd;
1565
1566 pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld ch %p sess %p\n",
1567 srp_tsk->tsk_mgmt_func, srp_tsk->task_tag, srp_tsk->tag, ch,
1568 ch->sess);
1569
1570 srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
1571 send_ioctx->cmd.tag = srp_tsk->tag;
1572 tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
1573 rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1574 scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1575 GFP_KERNEL, srp_tsk->task_tag,
1576 TARGET_SCF_ACK_KREF);
1577 if (rc != 0) {
1578 send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1579 cmd->se_tfo->queue_tm_rsp(cmd);
1580 }
1581 return;
1582}
1583
1584
1585
1586
1587
1588
1589static bool
1590srpt_handle_new_iu(struct srpt_rdma_ch *ch, struct srpt_recv_ioctx *recv_ioctx)
1591{
1592 struct srpt_send_ioctx *send_ioctx = NULL;
1593 struct srp_cmd *srp_cmd;
1594 bool res = false;
1595 u8 opcode;
1596
1597 BUG_ON(!ch);
1598 BUG_ON(!recv_ioctx);
1599
1600 if (unlikely(ch->state == CH_CONNECTING))
1601 goto push;
1602
1603 ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1604 recv_ioctx->ioctx.dma,
1605 recv_ioctx->ioctx.offset + srp_max_req_size,
1606 DMA_FROM_DEVICE);
1607
1608 srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1609 opcode = srp_cmd->opcode;
1610 if (opcode == SRP_CMD || opcode == SRP_TSK_MGMT) {
1611 send_ioctx = srpt_get_send_ioctx(ch);
1612 if (unlikely(!send_ioctx))
1613 goto push;
1614 }
1615
1616 if (!list_empty(&recv_ioctx->wait_list)) {
1617 WARN_ON_ONCE(!ch->processing_wait_list);
1618 list_del_init(&recv_ioctx->wait_list);
1619 }
1620
1621 switch (opcode) {
1622 case SRP_CMD:
1623 srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1624 break;
1625 case SRP_TSK_MGMT:
1626 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1627 break;
1628 case SRP_I_LOGOUT:
1629 pr_err("Not yet implemented: SRP_I_LOGOUT\n");
1630 break;
1631 case SRP_CRED_RSP:
1632 pr_debug("received SRP_CRED_RSP\n");
1633 break;
1634 case SRP_AER_RSP:
1635 pr_debug("received SRP_AER_RSP\n");
1636 break;
1637 case SRP_RSP:
1638 pr_err("Received SRP_RSP\n");
1639 break;
1640 default:
1641 pr_err("received IU with unknown opcode 0x%x\n", opcode);
1642 break;
1643 }
1644
1645 if (!send_ioctx || !send_ioctx->recv_ioctx)
1646 srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
1647 res = true;
1648
1649out:
1650 return res;
1651
1652push:
1653 if (list_empty(&recv_ioctx->wait_list)) {
1654 WARN_ON_ONCE(ch->processing_wait_list);
1655 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
1656 }
1657 goto out;
1658}
1659
1660static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1661{
1662 struct srpt_rdma_ch *ch = cq->cq_context;
1663 struct srpt_recv_ioctx *ioctx =
1664 container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
1665
1666 if (wc->status == IB_WC_SUCCESS) {
1667 int req_lim;
1668
1669 req_lim = atomic_dec_return(&ch->req_lim);
1670 if (unlikely(req_lim < 0))
1671 pr_err("req_lim = %d < 0\n", req_lim);
1672 ioctx->byte_len = wc->byte_len;
1673 srpt_handle_new_iu(ch, ioctx);
1674 } else {
1675 pr_info_ratelimited("receiving failed for ioctx %p with status %d\n",
1676 ioctx, wc->status);
1677 }
1678}
1679
1680
1681
1682
1683
1684
1685static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1686{
1687 struct srpt_recv_ioctx *recv_ioctx, *tmp;
1688
1689 WARN_ON_ONCE(ch->state == CH_CONNECTING);
1690
1691 if (list_empty(&ch->cmd_wait_list))
1692 return;
1693
1694 WARN_ON_ONCE(ch->processing_wait_list);
1695 ch->processing_wait_list = true;
1696 list_for_each_entry_safe(recv_ioctx, tmp, &ch->cmd_wait_list,
1697 wait_list) {
1698 if (!srpt_handle_new_iu(ch, recv_ioctx))
1699 break;
1700 }
1701 ch->processing_wait_list = false;
1702}
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
1722{
1723 struct srpt_rdma_ch *ch = cq->cq_context;
1724 struct srpt_send_ioctx *ioctx =
1725 container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1726 enum srpt_command_state state;
1727
1728 state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1729
1730 WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1731 state != SRPT_STATE_MGMT_RSP_SENT);
1732
1733 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
1734
1735 if (wc->status != IB_WC_SUCCESS)
1736 pr_info("sending response for ioctx 0x%p failed with status %d\n",
1737 ioctx, wc->status);
1738
1739 if (state != SRPT_STATE_DONE) {
1740 transport_generic_free_cmd(&ioctx->cmd, 0);
1741 } else {
1742 pr_err("IB completion has been received too late for wr_id = %u.\n",
1743 ioctx->ioctx.index);
1744 }
1745
1746 srpt_process_wait_list(ch);
1747}
1748
1749
1750
1751
1752
1753static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1754{
1755 struct ib_qp_init_attr *qp_init;
1756 struct srpt_port *sport = ch->sport;
1757 struct srpt_device *sdev = sport->sdev;
1758 const struct ib_device_attr *attrs = &sdev->device->attrs;
1759 int sq_size = sport->port_attrib.srp_sq_size;
1760 int i, ret;
1761
1762 WARN_ON(ch->rq_size < 1);
1763
1764 ret = -ENOMEM;
1765 qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
1766 if (!qp_init)
1767 goto out;
1768
1769retry:
1770 ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + sq_size,
1771 0 , IB_POLL_WORKQUEUE);
1772 if (IS_ERR(ch->cq)) {
1773 ret = PTR_ERR(ch->cq);
1774 pr_err("failed to create CQ cqe= %d ret= %d\n",
1775 ch->rq_size + sq_size, ret);
1776 goto out;
1777 }
1778
1779 qp_init->qp_context = (void *)ch;
1780 qp_init->event_handler
1781 = (void(*)(struct ib_event *, void*))srpt_qp_event;
1782 qp_init->send_cq = ch->cq;
1783 qp_init->recv_cq = ch->cq;
1784 qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1785 qp_init->qp_type = IB_QPT_RC;
1786
1787
1788
1789
1790
1791
1792
1793 qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
1794 qp_init->cap.max_rdma_ctxs = sq_size / 2;
1795 qp_init->cap.max_send_sge = min(attrs->max_send_sge,
1796 SRPT_MAX_SG_PER_WQE);
1797 qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
1798 SRPT_MAX_SG_PER_WQE);
1799 qp_init->port_num = ch->sport->port;
1800 if (sdev->use_srq) {
1801 qp_init->srq = sdev->srq;
1802 } else {
1803 qp_init->cap.max_recv_wr = ch->rq_size;
1804 qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
1805 SRPT_MAX_SG_PER_WQE);
1806 }
1807
1808 if (ch->using_rdma_cm) {
1809 ret = rdma_create_qp(ch->rdma_cm.cm_id, sdev->pd, qp_init);
1810 ch->qp = ch->rdma_cm.cm_id->qp;
1811 } else {
1812 ch->qp = ib_create_qp(sdev->pd, qp_init);
1813 if (!IS_ERR(ch->qp)) {
1814 ret = srpt_init_ch_qp(ch, ch->qp);
1815 if (ret)
1816 ib_destroy_qp(ch->qp);
1817 } else {
1818 ret = PTR_ERR(ch->qp);
1819 }
1820 }
1821 if (ret) {
1822 bool retry = sq_size > MIN_SRPT_SQ_SIZE;
1823
1824 if (retry) {
1825 pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
1826 sq_size, ret);
1827 ib_free_cq(ch->cq);
1828 sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
1829 goto retry;
1830 } else {
1831 pr_err("failed to create queue pair with sq_size = %d (%d)\n",
1832 sq_size, ret);
1833 goto err_destroy_cq;
1834 }
1835 }
1836
1837 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1838
1839 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d ch= %p\n",
1840 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1841 qp_init->cap.max_send_wr, ch);
1842
1843 if (!sdev->use_srq)
1844 for (i = 0; i < ch->rq_size; i++)
1845 srpt_post_recv(sdev, ch, ch->ioctx_recv_ring[i]);
1846
1847out:
1848 kfree(qp_init);
1849 return ret;
1850
1851err_destroy_cq:
1852 ch->qp = NULL;
1853 ib_free_cq(ch->cq);
1854 goto out;
1855}
1856
1857static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1858{
1859 ib_destroy_qp(ch->qp);
1860 ib_free_cq(ch->cq);
1861}
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873static bool srpt_close_ch(struct srpt_rdma_ch *ch)
1874{
1875 int ret;
1876
1877 if (!srpt_set_ch_state(ch, CH_DRAINING)) {
1878 pr_debug("%s: already closed\n", ch->sess_name);
1879 return false;
1880 }
1881
1882 kref_get(&ch->kref);
1883
1884 ret = srpt_ch_qp_err(ch);
1885 if (ret < 0)
1886 pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1887 ch->sess_name, ch->qp->qp_num, ret);
1888
1889 ret = srpt_zerolength_write(ch);
1890 if (ret < 0) {
1891 pr_err("%s-%d: queuing zero-length write failed: %d\n",
1892 ch->sess_name, ch->qp->qp_num, ret);
1893 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
1894 schedule_work(&ch->release_work);
1895 else
1896 WARN_ON_ONCE(true);
1897 }
1898
1899 kref_put(&ch->kref, srpt_free_ch);
1900
1901 return true;
1902}
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
1914{
1915 int ret;
1916
1917 if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
1918 return -ENOTCONN;
1919
1920 if (ch->using_rdma_cm) {
1921 ret = rdma_disconnect(ch->rdma_cm.cm_id);
1922 } else {
1923 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, NULL, 0);
1924 if (ret < 0)
1925 ret = ib_send_cm_drep(ch->ib_cm.cm_id, NULL, 0);
1926 }
1927
1928 if (ret < 0 && srpt_close_ch(ch))
1929 ret = 0;
1930
1931 return ret;
1932}
1933
1934static bool srpt_ch_closed(struct srpt_port *sport, struct srpt_rdma_ch *ch)
1935{
1936 struct srpt_nexus *nexus;
1937 struct srpt_rdma_ch *ch2;
1938 bool res = true;
1939
1940 rcu_read_lock();
1941 list_for_each_entry(nexus, &sport->nexus_list, entry) {
1942 list_for_each_entry(ch2, &nexus->ch_list, list) {
1943 if (ch2 == ch) {
1944 res = false;
1945 goto done;
1946 }
1947 }
1948 }
1949done:
1950 rcu_read_unlock();
1951
1952 return res;
1953}
1954
1955
1956static void srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch)
1957{
1958 struct srpt_port *sport = ch->sport;
1959
1960 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
1961 ch->state);
1962
1963 mutex_lock(&sport->mutex);
1964 srpt_disconnect_ch(ch);
1965 mutex_unlock(&sport->mutex);
1966
1967 while (wait_event_timeout(sport->ch_releaseQ, srpt_ch_closed(sport, ch),
1968 5 * HZ) == 0)
1969 pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
1970 ch->sess_name, ch->qp->qp_num, ch->state);
1971
1972}
1973
1974static void __srpt_close_all_ch(struct srpt_port *sport)
1975{
1976 struct srpt_nexus *nexus;
1977 struct srpt_rdma_ch *ch;
1978
1979 lockdep_assert_held(&sport->mutex);
1980
1981 list_for_each_entry(nexus, &sport->nexus_list, entry) {
1982 list_for_each_entry(ch, &nexus->ch_list, list) {
1983 if (srpt_disconnect_ch(ch) >= 0)
1984 pr_info("Closing channel %s because target %s_%d has been disabled\n",
1985 ch->sess_name,
1986 dev_name(&sport->sdev->device->dev),
1987 sport->port);
1988 srpt_close_ch(ch);
1989 }
1990 }
1991}
1992
1993
1994
1995
1996
1997static struct srpt_nexus *srpt_get_nexus(struct srpt_port *sport,
1998 const u8 i_port_id[16],
1999 const u8 t_port_id[16])
2000{
2001 struct srpt_nexus *nexus = NULL, *tmp_nexus = NULL, *n;
2002
2003 for (;;) {
2004 mutex_lock(&sport->mutex);
2005 list_for_each_entry(n, &sport->nexus_list, entry) {
2006 if (memcmp(n->i_port_id, i_port_id, 16) == 0 &&
2007 memcmp(n->t_port_id, t_port_id, 16) == 0) {
2008 nexus = n;
2009 break;
2010 }
2011 }
2012 if (!nexus && tmp_nexus) {
2013 list_add_tail_rcu(&tmp_nexus->entry,
2014 &sport->nexus_list);
2015 swap(nexus, tmp_nexus);
2016 }
2017 mutex_unlock(&sport->mutex);
2018
2019 if (nexus)
2020 break;
2021 tmp_nexus = kzalloc(sizeof(*nexus), GFP_KERNEL);
2022 if (!tmp_nexus) {
2023 nexus = ERR_PTR(-ENOMEM);
2024 break;
2025 }
2026 INIT_LIST_HEAD(&tmp_nexus->ch_list);
2027 memcpy(tmp_nexus->i_port_id, i_port_id, 16);
2028 memcpy(tmp_nexus->t_port_id, t_port_id, 16);
2029 }
2030
2031 kfree(tmp_nexus);
2032
2033 return nexus;
2034}
2035
2036static void srpt_set_enabled(struct srpt_port *sport, bool enabled)
2037 __must_hold(&sport->mutex)
2038{
2039 lockdep_assert_held(&sport->mutex);
2040
2041 if (sport->enabled == enabled)
2042 return;
2043 sport->enabled = enabled;
2044 if (!enabled)
2045 __srpt_close_all_ch(sport);
2046}
2047
2048static void srpt_free_ch(struct kref *kref)
2049{
2050 struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
2051
2052 kfree_rcu(ch, rcu);
2053}
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063static void srpt_release_channel_work(struct work_struct *w)
2064{
2065 struct srpt_rdma_ch *ch;
2066 struct srpt_device *sdev;
2067 struct srpt_port *sport;
2068 struct se_session *se_sess;
2069
2070 ch = container_of(w, struct srpt_rdma_ch, release_work);
2071 pr_debug("%s-%d\n", ch->sess_name, ch->qp->qp_num);
2072
2073 sdev = ch->sport->sdev;
2074 BUG_ON(!sdev);
2075
2076 se_sess = ch->sess;
2077 BUG_ON(!se_sess);
2078
2079 target_sess_cmd_list_set_waiting(se_sess);
2080 target_wait_for_sess_cmds(se_sess);
2081
2082 target_remove_session(se_sess);
2083 ch->sess = NULL;
2084
2085 if (ch->using_rdma_cm)
2086 rdma_destroy_id(ch->rdma_cm.cm_id);
2087 else
2088 ib_destroy_cm_id(ch->ib_cm.cm_id);
2089
2090 sport = ch->sport;
2091 mutex_lock(&sport->mutex);
2092 list_del_rcu(&ch->list);
2093 mutex_unlock(&sport->mutex);
2094
2095 srpt_destroy_ch_ib(ch);
2096
2097 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2098 ch->sport->sdev, ch->rq_size,
2099 ch->rsp_buf_cache, DMA_TO_DEVICE);
2100
2101 kmem_cache_destroy(ch->rsp_buf_cache);
2102
2103 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring,
2104 sdev, ch->rq_size,
2105 ch->req_buf_cache, DMA_FROM_DEVICE);
2106
2107 kmem_cache_destroy(ch->req_buf_cache);
2108
2109 wake_up(&sport->ch_releaseQ);
2110
2111 kref_put(&ch->kref, srpt_free_ch);
2112}
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128static int srpt_cm_req_recv(struct srpt_device *const sdev,
2129 struct ib_cm_id *ib_cm_id,
2130 struct rdma_cm_id *rdma_cm_id,
2131 u8 port_num, __be16 pkey,
2132 const struct srp_login_req *req,
2133 const char *src_addr)
2134{
2135 struct srpt_port *sport = &sdev->port[port_num - 1];
2136 struct srpt_nexus *nexus;
2137 struct srp_login_rsp *rsp = NULL;
2138 struct srp_login_rej *rej = NULL;
2139 union {
2140 struct rdma_conn_param rdma_cm;
2141 struct ib_cm_rep_param ib_cm;
2142 } *rep_param = NULL;
2143 struct srpt_rdma_ch *ch = NULL;
2144 char i_port_id[36];
2145 u32 it_iu_len;
2146 int i, tag_num, tag_size, ret;
2147
2148 WARN_ON_ONCE(irqs_disabled());
2149
2150 if (WARN_ON(!sdev || !req))
2151 return -EINVAL;
2152
2153 it_iu_len = be32_to_cpu(req->req_it_iu_len);
2154
2155 pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n",
2156 req->initiator_port_id, req->target_port_id, it_iu_len,
2157 port_num, &sport->gid, be16_to_cpu(pkey));
2158
2159 nexus = srpt_get_nexus(sport, req->initiator_port_id,
2160 req->target_port_id);
2161 if (IS_ERR(nexus)) {
2162 ret = PTR_ERR(nexus);
2163 goto out;
2164 }
2165
2166 ret = -ENOMEM;
2167 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
2168 rej = kzalloc(sizeof(*rej), GFP_KERNEL);
2169 rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
2170 if (!rsp || !rej || !rep_param)
2171 goto out;
2172
2173 ret = -EINVAL;
2174 if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
2175 rej->reason = cpu_to_be32(
2176 SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
2177 pr_err("rejected SRP_LOGIN_REQ because its length (%d bytes) is out of range (%d .. %d)\n",
2178 it_iu_len, 64, srp_max_req_size);
2179 goto reject;
2180 }
2181
2182 if (!sport->enabled) {
2183 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2184 pr_info("rejected SRP_LOGIN_REQ because target port %s_%d has not yet been enabled\n",
2185 dev_name(&sport->sdev->device->dev), port_num);
2186 goto reject;
2187 }
2188
2189 if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
2190 || *(__be64 *)(req->target_port_id + 8) !=
2191 cpu_to_be64(srpt_service_guid)) {
2192 rej->reason = cpu_to_be32(
2193 SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
2194 pr_err("rejected SRP_LOGIN_REQ because it has an invalid target port identifier.\n");
2195 goto reject;
2196 }
2197
2198 ret = -ENOMEM;
2199 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
2200 if (!ch) {
2201 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2202 pr_err("rejected SRP_LOGIN_REQ because out of memory.\n");
2203 goto reject;
2204 }
2205
2206 kref_init(&ch->kref);
2207 ch->pkey = be16_to_cpu(pkey);
2208 ch->nexus = nexus;
2209 ch->zw_cqe.done = srpt_zerolength_write_done;
2210 INIT_WORK(&ch->release_work, srpt_release_channel_work);
2211 ch->sport = sport;
2212 if (ib_cm_id) {
2213 ch->ib_cm.cm_id = ib_cm_id;
2214 ib_cm_id->context = ch;
2215 } else {
2216 ch->using_rdma_cm = true;
2217 ch->rdma_cm.cm_id = rdma_cm_id;
2218 rdma_cm_id->context = ch;
2219 }
2220
2221
2222
2223
2224
2225 ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr);
2226 spin_lock_init(&ch->spinlock);
2227 ch->state = CH_CONNECTING;
2228 INIT_LIST_HEAD(&ch->cmd_wait_list);
2229 ch->max_rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
2230
2231 ch->rsp_buf_cache = kmem_cache_create("srpt-rsp-buf", ch->max_rsp_size,
2232 512, 0, NULL);
2233 if (!ch->rsp_buf_cache)
2234 goto free_ch;
2235
2236 ch->ioctx_ring = (struct srpt_send_ioctx **)
2237 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2238 sizeof(*ch->ioctx_ring[0]),
2239 ch->rsp_buf_cache, 0, DMA_TO_DEVICE);
2240 if (!ch->ioctx_ring) {
2241 pr_err("rejected SRP_LOGIN_REQ because creating a new QP SQ ring failed.\n");
2242 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2243 goto free_rsp_cache;
2244 }
2245
2246 for (i = 0; i < ch->rq_size; i++)
2247 ch->ioctx_ring[i]->ch = ch;
2248 if (!sdev->use_srq) {
2249 u16 imm_data_offset = req->req_flags & SRP_IMMED_REQUESTED ?
2250 be16_to_cpu(req->imm_data_offset) : 0;
2251 u16 alignment_offset;
2252 u32 req_sz;
2253
2254 if (req->req_flags & SRP_IMMED_REQUESTED)
2255 pr_debug("imm_data_offset = %d\n",
2256 be16_to_cpu(req->imm_data_offset));
2257 if (imm_data_offset >= sizeof(struct srp_cmd)) {
2258 ch->imm_data_offset = imm_data_offset;
2259 rsp->rsp_flags |= SRP_LOGIN_RSP_IMMED_SUPP;
2260 } else {
2261 ch->imm_data_offset = 0;
2262 }
2263 alignment_offset = round_up(imm_data_offset, 512) -
2264 imm_data_offset;
2265 req_sz = alignment_offset + imm_data_offset + srp_max_req_size;
2266 ch->req_buf_cache = kmem_cache_create("srpt-req-buf", req_sz,
2267 512, 0, NULL);
2268 if (!ch->req_buf_cache)
2269 goto free_rsp_ring;
2270
2271 ch->ioctx_recv_ring = (struct srpt_recv_ioctx **)
2272 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2273 sizeof(*ch->ioctx_recv_ring[0]),
2274 ch->req_buf_cache,
2275 alignment_offset,
2276 DMA_FROM_DEVICE);
2277 if (!ch->ioctx_recv_ring) {
2278 pr_err("rejected SRP_LOGIN_REQ because creating a new QP RQ ring failed.\n");
2279 rej->reason =
2280 cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2281 goto free_recv_cache;
2282 }
2283 for (i = 0; i < ch->rq_size; i++)
2284 INIT_LIST_HEAD(&ch->ioctx_recv_ring[i]->wait_list);
2285 }
2286
2287 ret = srpt_create_ch_ib(ch);
2288 if (ret) {
2289 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2290 pr_err("rejected SRP_LOGIN_REQ because creating a new RDMA channel failed.\n");
2291 goto free_recv_ring;
2292 }
2293
2294 strlcpy(ch->sess_name, src_addr, sizeof(ch->sess_name));
2295 snprintf(i_port_id, sizeof(i_port_id), "0x%016llx%016llx",
2296 be64_to_cpu(*(__be64 *)nexus->i_port_id),
2297 be64_to_cpu(*(__be64 *)(nexus->i_port_id + 8)));
2298
2299 pr_debug("registering session %s\n", ch->sess_name);
2300
2301 tag_num = ch->rq_size;
2302 tag_size = 1;
2303 if (sport->port_guid_tpg.se_tpg_wwn)
2304 ch->sess = target_setup_session(&sport->port_guid_tpg, tag_num,
2305 tag_size, TARGET_PROT_NORMAL,
2306 ch->sess_name, ch, NULL);
2307 if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
2308 ch->sess = target_setup_session(&sport->port_gid_tpg, tag_num,
2309 tag_size, TARGET_PROT_NORMAL, i_port_id,
2310 ch, NULL);
2311
2312 if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
2313 ch->sess = target_setup_session(&sport->port_gid_tpg, tag_num,
2314 tag_size, TARGET_PROT_NORMAL,
2315 i_port_id + 2, ch, NULL);
2316 if (IS_ERR_OR_NULL(ch->sess)) {
2317 WARN_ON_ONCE(ch->sess == NULL);
2318 ret = PTR_ERR(ch->sess);
2319 ch->sess = NULL;
2320 pr_info("Rejected login for initiator %s: ret = %d.\n",
2321 ch->sess_name, ret);
2322 rej->reason = cpu_to_be32(ret == -ENOMEM ?
2323 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
2324 SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
2325 goto destroy_ib;
2326 }
2327
2328 mutex_lock(&sport->mutex);
2329
2330 if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
2331 struct srpt_rdma_ch *ch2;
2332
2333 list_for_each_entry(ch2, &nexus->ch_list, list) {
2334 if (srpt_disconnect_ch(ch2) < 0)
2335 continue;
2336 pr_info("Relogin - closed existing channel %s\n",
2337 ch2->sess_name);
2338 rsp->rsp_flags |= SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
2339 }
2340 } else {
2341 rsp->rsp_flags |= SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
2342 }
2343
2344 list_add_tail_rcu(&ch->list, &nexus->ch_list);
2345
2346 if (!sport->enabled) {
2347 rej->reason = cpu_to_be32(
2348 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2349 pr_info("rejected SRP_LOGIN_REQ because target %s_%d is not enabled\n",
2350 dev_name(&sdev->device->dev), port_num);
2351 mutex_unlock(&sport->mutex);
2352 goto reject;
2353 }
2354
2355 mutex_unlock(&sport->mutex);
2356
2357 ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rtr(ch, ch->qp);
2358 if (ret) {
2359 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2360 pr_err("rejected SRP_LOGIN_REQ because enabling RTR failed (error code = %d)\n",
2361 ret);
2362 goto reject;
2363 }
2364
2365 pr_debug("Establish connection sess=%p name=%s ch=%p\n", ch->sess,
2366 ch->sess_name, ch);
2367
2368
2369 rsp->opcode = SRP_LOGIN_RSP;
2370 rsp->tag = req->tag;
2371 rsp->max_it_iu_len = cpu_to_be32(srp_max_req_size);
2372 rsp->max_ti_iu_len = req->req_it_iu_len;
2373 ch->max_ti_iu_len = it_iu_len;
2374 rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
2375 SRP_BUF_FORMAT_INDIRECT);
2376 rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2377 atomic_set(&ch->req_lim, ch->rq_size);
2378 atomic_set(&ch->req_lim_delta, 0);
2379
2380
2381 if (ch->using_rdma_cm) {
2382 rep_param->rdma_cm.private_data = (void *)rsp;
2383 rep_param->rdma_cm.private_data_len = sizeof(*rsp);
2384 rep_param->rdma_cm.rnr_retry_count = 7;
2385 rep_param->rdma_cm.flow_control = 1;
2386 rep_param->rdma_cm.responder_resources = 4;
2387 rep_param->rdma_cm.initiator_depth = 4;
2388 } else {
2389 rep_param->ib_cm.qp_num = ch->qp->qp_num;
2390 rep_param->ib_cm.private_data = (void *)rsp;
2391 rep_param->ib_cm.private_data_len = sizeof(*rsp);
2392 rep_param->ib_cm.rnr_retry_count = 7;
2393 rep_param->ib_cm.flow_control = 1;
2394 rep_param->ib_cm.failover_accepted = 0;
2395 rep_param->ib_cm.srq = 1;
2396 rep_param->ib_cm.responder_resources = 4;
2397 rep_param->ib_cm.initiator_depth = 4;
2398 }
2399
2400
2401
2402
2403
2404 mutex_lock(&sport->mutex);
2405 if (sport->enabled && ch->state == CH_CONNECTING) {
2406 if (ch->using_rdma_cm)
2407 ret = rdma_accept(rdma_cm_id, &rep_param->rdma_cm);
2408 else
2409 ret = ib_send_cm_rep(ib_cm_id, &rep_param->ib_cm);
2410 } else {
2411 ret = -EINVAL;
2412 }
2413 mutex_unlock(&sport->mutex);
2414
2415 switch (ret) {
2416 case 0:
2417 break;
2418 case -EINVAL:
2419 goto reject;
2420 default:
2421 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2422 pr_err("sending SRP_LOGIN_REQ response failed (error code = %d)\n",
2423 ret);
2424 goto reject;
2425 }
2426
2427 goto out;
2428
2429destroy_ib:
2430 srpt_destroy_ch_ib(ch);
2431
2432free_recv_ring:
2433 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring,
2434 ch->sport->sdev, ch->rq_size,
2435 ch->req_buf_cache, DMA_FROM_DEVICE);
2436
2437free_recv_cache:
2438 kmem_cache_destroy(ch->req_buf_cache);
2439
2440free_rsp_ring:
2441 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2442 ch->sport->sdev, ch->rq_size,
2443 ch->rsp_buf_cache, DMA_TO_DEVICE);
2444
2445free_rsp_cache:
2446 kmem_cache_destroy(ch->rsp_buf_cache);
2447
2448free_ch:
2449 if (rdma_cm_id)
2450 rdma_cm_id->context = NULL;
2451 else
2452 ib_cm_id->context = NULL;
2453 kfree(ch);
2454 ch = NULL;
2455
2456 WARN_ON_ONCE(ret == 0);
2457
2458reject:
2459 pr_info("Rejecting login with reason %#x\n", be32_to_cpu(rej->reason));
2460 rej->opcode = SRP_LOGIN_REJ;
2461 rej->tag = req->tag;
2462 rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
2463 SRP_BUF_FORMAT_INDIRECT);
2464
2465 if (rdma_cm_id)
2466 rdma_reject(rdma_cm_id, rej, sizeof(*rej));
2467 else
2468 ib_send_cm_rej(ib_cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2469 rej, sizeof(*rej));
2470
2471 if (ch && ch->sess) {
2472 srpt_close_ch(ch);
2473
2474
2475
2476
2477 ret = 0;
2478 }
2479
2480out:
2481 kfree(rep_param);
2482 kfree(rsp);
2483 kfree(rej);
2484
2485 return ret;
2486}
2487
2488static int srpt_ib_cm_req_recv(struct ib_cm_id *cm_id,
2489 const struct ib_cm_req_event_param *param,
2490 void *private_data)
2491{
2492 char sguid[40];
2493
2494 srpt_format_guid(sguid, sizeof(sguid),
2495 ¶m->primary_path->dgid.global.interface_id);
2496
2497 return srpt_cm_req_recv(cm_id->context, cm_id, NULL, param->port,
2498 param->primary_path->pkey,
2499 private_data, sguid);
2500}
2501
2502static int srpt_rdma_cm_req_recv(struct rdma_cm_id *cm_id,
2503 struct rdma_cm_event *event)
2504{
2505 struct srpt_device *sdev;
2506 struct srp_login_req req;
2507 const struct srp_login_req_rdma *req_rdma;
2508 char src_addr[40];
2509
2510 sdev = ib_get_client_data(cm_id->device, &srpt_client);
2511 if (!sdev)
2512 return -ECONNREFUSED;
2513
2514 if (event->param.conn.private_data_len < sizeof(*req_rdma))
2515 return -EINVAL;
2516
2517
2518 req_rdma = event->param.conn.private_data;
2519 memset(&req, 0, sizeof(req));
2520 req.opcode = req_rdma->opcode;
2521 req.tag = req_rdma->tag;
2522 req.req_it_iu_len = req_rdma->req_it_iu_len;
2523 req.req_buf_fmt = req_rdma->req_buf_fmt;
2524 req.req_flags = req_rdma->req_flags;
2525 memcpy(req.initiator_port_id, req_rdma->initiator_port_id, 16);
2526 memcpy(req.target_port_id, req_rdma->target_port_id, 16);
2527 req.imm_data_offset = req_rdma->imm_data_offset;
2528
2529 snprintf(src_addr, sizeof(src_addr), "%pIS",
2530 &cm_id->route.addr.src_addr);
2531
2532 return srpt_cm_req_recv(sdev, NULL, cm_id, cm_id->port_num,
2533 cm_id->route.path_rec->pkey, &req, src_addr);
2534}
2535
2536static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2537 enum ib_cm_rej_reason reason,
2538 const u8 *private_data,
2539 u8 private_data_len)
2540{
2541 char *priv = NULL;
2542 int i;
2543
2544 if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2545 GFP_KERNEL))) {
2546 for (i = 0; i < private_data_len; i++)
2547 sprintf(priv + 3 * i, " %02x", private_data[i]);
2548 }
2549 pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2550 ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2551 "; private data" : "", priv ? priv : " (?)");
2552 kfree(priv);
2553}
2554
2555
2556
2557
2558
2559
2560
2561
2562static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
2563{
2564 int ret;
2565
2566 ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rts(ch, ch->qp);
2567 if (ret < 0) {
2568 pr_err("%s-%d: QP transition to RTS failed\n", ch->sess_name,
2569 ch->qp->qp_num);
2570 srpt_close_ch(ch);
2571 return;
2572 }
2573
2574
2575
2576
2577
2578
2579 if (!srpt_set_ch_state(ch, CH_LIVE)) {
2580 pr_err("%s-%d: channel transition to LIVE state failed\n",
2581 ch->sess_name, ch->qp->qp_num);
2582 return;
2583 }
2584
2585
2586 ret = srpt_zerolength_write(ch);
2587 WARN_ONCE(ret < 0, "%d\n", ret);
2588}
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602static int srpt_cm_handler(struct ib_cm_id *cm_id,
2603 const struct ib_cm_event *event)
2604{
2605 struct srpt_rdma_ch *ch = cm_id->context;
2606 int ret;
2607
2608 ret = 0;
2609 switch (event->event) {
2610 case IB_CM_REQ_RECEIVED:
2611 ret = srpt_ib_cm_req_recv(cm_id, &event->param.req_rcvd,
2612 event->private_data);
2613 break;
2614 case IB_CM_REJ_RECEIVED:
2615 srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2616 event->private_data,
2617 IB_CM_REJ_PRIVATE_DATA_SIZE);
2618 break;
2619 case IB_CM_RTU_RECEIVED:
2620 case IB_CM_USER_ESTABLISHED:
2621 srpt_cm_rtu_recv(ch);
2622 break;
2623 case IB_CM_DREQ_RECEIVED:
2624 srpt_disconnect_ch(ch);
2625 break;
2626 case IB_CM_DREP_RECEIVED:
2627 pr_info("Received CM DREP message for ch %s-%d.\n",
2628 ch->sess_name, ch->qp->qp_num);
2629 srpt_close_ch(ch);
2630 break;
2631 case IB_CM_TIMEWAIT_EXIT:
2632 pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2633 ch->sess_name, ch->qp->qp_num);
2634 srpt_close_ch(ch);
2635 break;
2636 case IB_CM_REP_ERROR:
2637 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2638 ch->qp->qp_num);
2639 break;
2640 case IB_CM_DREQ_ERROR:
2641 pr_info("Received CM DREQ ERROR event.\n");
2642 break;
2643 case IB_CM_MRA_RECEIVED:
2644 pr_info("Received CM MRA event\n");
2645 break;
2646 default:
2647 pr_err("received unrecognized CM event %d\n", event->event);
2648 break;
2649 }
2650
2651 return ret;
2652}
2653
2654static int srpt_rdma_cm_handler(struct rdma_cm_id *cm_id,
2655 struct rdma_cm_event *event)
2656{
2657 struct srpt_rdma_ch *ch = cm_id->context;
2658 int ret = 0;
2659
2660 switch (event->event) {
2661 case RDMA_CM_EVENT_CONNECT_REQUEST:
2662 ret = srpt_rdma_cm_req_recv(cm_id, event);
2663 break;
2664 case RDMA_CM_EVENT_REJECTED:
2665 srpt_cm_rej_recv(ch, event->status,
2666 event->param.conn.private_data,
2667 event->param.conn.private_data_len);
2668 break;
2669 case RDMA_CM_EVENT_ESTABLISHED:
2670 srpt_cm_rtu_recv(ch);
2671 break;
2672 case RDMA_CM_EVENT_DISCONNECTED:
2673 if (ch->state < CH_DISCONNECTING)
2674 srpt_disconnect_ch(ch);
2675 else
2676 srpt_close_ch(ch);
2677 break;
2678 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
2679 srpt_close_ch(ch);
2680 break;
2681 case RDMA_CM_EVENT_UNREACHABLE:
2682 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2683 ch->qp->qp_num);
2684 break;
2685 case RDMA_CM_EVENT_DEVICE_REMOVAL:
2686 case RDMA_CM_EVENT_ADDR_CHANGE:
2687 break;
2688 default:
2689 pr_err("received unrecognized RDMA CM event %d\n",
2690 event->event);
2691 break;
2692 }
2693
2694 return ret;
2695}
2696
2697
2698
2699
2700static int srpt_write_pending(struct se_cmd *se_cmd)
2701{
2702 struct srpt_send_ioctx *ioctx =
2703 container_of(se_cmd, struct srpt_send_ioctx, cmd);
2704 struct srpt_rdma_ch *ch = ioctx->ch;
2705 struct ib_send_wr *first_wr = NULL;
2706 struct ib_cqe *cqe = &ioctx->rdma_cqe;
2707 enum srpt_command_state new_state;
2708 int ret, i;
2709
2710 if (ioctx->recv_ioctx) {
2711 srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);
2712 target_execute_cmd(&ioctx->cmd);
2713 return 0;
2714 }
2715
2716 new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2717 WARN_ON(new_state == SRPT_STATE_DONE);
2718
2719 if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {
2720 pr_warn("%s: IB send queue full (needed %d)\n",
2721 __func__, ioctx->n_rdma);
2722 ret = -ENOMEM;
2723 goto out_undo;
2724 }
2725
2726 cqe->done = srpt_rdma_read_done;
2727 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2728 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2729
2730 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port,
2731 cqe, first_wr);
2732 cqe = NULL;
2733 }
2734
2735 ret = ib_post_send(ch->qp, first_wr, NULL);
2736 if (ret) {
2737 pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2738 __func__, ret, ioctx->n_rdma,
2739 atomic_read(&ch->sq_wr_avail));
2740 goto out_undo;
2741 }
2742
2743 return 0;
2744out_undo:
2745 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
2746 return ret;
2747}
2748
2749static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2750{
2751 switch (tcm_mgmt_status) {
2752 case TMR_FUNCTION_COMPLETE:
2753 return SRP_TSK_MGMT_SUCCESS;
2754 case TMR_FUNCTION_REJECTED:
2755 return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2756 }
2757 return SRP_TSK_MGMT_FAILED;
2758}
2759
2760
2761
2762
2763
2764
2765
2766
2767static void srpt_queue_response(struct se_cmd *cmd)
2768{
2769 struct srpt_send_ioctx *ioctx =
2770 container_of(cmd, struct srpt_send_ioctx, cmd);
2771 struct srpt_rdma_ch *ch = ioctx->ch;
2772 struct srpt_device *sdev = ch->sport->sdev;
2773 struct ib_send_wr send_wr, *first_wr = &send_wr;
2774 struct ib_sge sge;
2775 enum srpt_command_state state;
2776 int resp_len, ret, i;
2777 u8 srp_tm_status;
2778
2779 BUG_ON(!ch);
2780
2781 state = ioctx->state;
2782 switch (state) {
2783 case SRPT_STATE_NEW:
2784 case SRPT_STATE_DATA_IN:
2785 ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2786 break;
2787 case SRPT_STATE_MGMT:
2788 ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2789 break;
2790 default:
2791 WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2792 ch, ioctx->ioctx.index, ioctx->state);
2793 break;
2794 }
2795
2796 if (WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))
2797 return;
2798
2799
2800 if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2801 ioctx->cmd.data_length &&
2802 !ioctx->queue_status_only) {
2803 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2804 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2805
2806 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
2807 ch->sport->port, NULL, first_wr);
2808 }
2809 }
2810
2811 if (state != SRPT_STATE_MGMT)
2812 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
2813 cmd->scsi_status);
2814 else {
2815 srp_tm_status
2816 = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2817 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
2818 ioctx->cmd.tag);
2819 }
2820
2821 atomic_inc(&ch->req_lim);
2822
2823 if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2824 &ch->sq_wr_avail) < 0)) {
2825 pr_warn("%s: IB send queue full (needed %d)\n",
2826 __func__, ioctx->n_rdma);
2827 ret = -ENOMEM;
2828 goto out;
2829 }
2830
2831 ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,
2832 DMA_TO_DEVICE);
2833
2834 sge.addr = ioctx->ioctx.dma;
2835 sge.length = resp_len;
2836 sge.lkey = sdev->lkey;
2837
2838 ioctx->ioctx.cqe.done = srpt_send_done;
2839 send_wr.next = NULL;
2840 send_wr.wr_cqe = &ioctx->ioctx.cqe;
2841 send_wr.sg_list = &sge;
2842 send_wr.num_sge = 1;
2843 send_wr.opcode = IB_WR_SEND;
2844 send_wr.send_flags = IB_SEND_SIGNALED;
2845
2846 ret = ib_post_send(ch->qp, first_wr, NULL);
2847 if (ret < 0) {
2848 pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2849 __func__, ioctx->cmd.tag, ret);
2850 goto out;
2851 }
2852
2853 return;
2854
2855out:
2856 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
2857 atomic_dec(&ch->req_lim);
2858 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
2859 target_put_sess_cmd(&ioctx->cmd);
2860}
2861
2862static int srpt_queue_data_in(struct se_cmd *cmd)
2863{
2864 srpt_queue_response(cmd);
2865 return 0;
2866}
2867
2868static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2869{
2870 srpt_queue_response(cmd);
2871}
2872
2873
2874
2875
2876
2877
2878
2879static void srpt_aborted_task(struct se_cmd *cmd)
2880{
2881 struct srpt_send_ioctx *ioctx = container_of(cmd,
2882 struct srpt_send_ioctx, cmd);
2883 struct srpt_rdma_ch *ch = ioctx->ch;
2884
2885 atomic_inc(&ch->req_lim_delta);
2886}
2887
2888static int srpt_queue_status(struct se_cmd *cmd)
2889{
2890 struct srpt_send_ioctx *ioctx;
2891
2892 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2893 BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2894 if (cmd->se_cmd_flags &
2895 (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2896 WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2897 ioctx->queue_status_only = true;
2898 srpt_queue_response(cmd);
2899 return 0;
2900}
2901
2902static void srpt_refresh_port_work(struct work_struct *work)
2903{
2904 struct srpt_port *sport = container_of(work, struct srpt_port, work);
2905
2906 srpt_refresh_port(sport);
2907}
2908
2909static bool srpt_ch_list_empty(struct srpt_port *sport)
2910{
2911 struct srpt_nexus *nexus;
2912 bool res = true;
2913
2914 rcu_read_lock();
2915 list_for_each_entry(nexus, &sport->nexus_list, entry)
2916 if (!list_empty(&nexus->ch_list))
2917 res = false;
2918 rcu_read_unlock();
2919
2920 return res;
2921}
2922
2923
2924
2925
2926
2927static int srpt_release_sport(struct srpt_port *sport)
2928{
2929 struct srpt_nexus *nexus, *next_n;
2930 struct srpt_rdma_ch *ch;
2931
2932 WARN_ON_ONCE(irqs_disabled());
2933
2934 mutex_lock(&sport->mutex);
2935 srpt_set_enabled(sport, false);
2936 mutex_unlock(&sport->mutex);
2937
2938 while (wait_event_timeout(sport->ch_releaseQ,
2939 srpt_ch_list_empty(sport), 5 * HZ) <= 0) {
2940 pr_info("%s_%d: waiting for session unregistration ...\n",
2941 dev_name(&sport->sdev->device->dev), sport->port);
2942 rcu_read_lock();
2943 list_for_each_entry(nexus, &sport->nexus_list, entry) {
2944 list_for_each_entry(ch, &nexus->ch_list, list) {
2945 pr_info("%s-%d: state %s\n",
2946 ch->sess_name, ch->qp->qp_num,
2947 get_ch_state_name(ch->state));
2948 }
2949 }
2950 rcu_read_unlock();
2951 }
2952
2953 mutex_lock(&sport->mutex);
2954 list_for_each_entry_safe(nexus, next_n, &sport->nexus_list, entry) {
2955 list_del(&nexus->entry);
2956 kfree_rcu(nexus, rcu);
2957 }
2958 mutex_unlock(&sport->mutex);
2959
2960 return 0;
2961}
2962
2963static struct se_wwn *__srpt_lookup_wwn(const char *name)
2964{
2965 struct ib_device *dev;
2966 struct srpt_device *sdev;
2967 struct srpt_port *sport;
2968 int i;
2969
2970 list_for_each_entry(sdev, &srpt_dev_list, list) {
2971 dev = sdev->device;
2972 if (!dev)
2973 continue;
2974
2975 for (i = 0; i < dev->phys_port_cnt; i++) {
2976 sport = &sdev->port[i];
2977
2978 if (strcmp(sport->port_guid, name) == 0)
2979 return &sport->port_guid_wwn;
2980 if (strcmp(sport->port_gid, name) == 0)
2981 return &sport->port_gid_wwn;
2982 }
2983 }
2984
2985 return NULL;
2986}
2987
2988static struct se_wwn *srpt_lookup_wwn(const char *name)
2989{
2990 struct se_wwn *wwn;
2991
2992 spin_lock(&srpt_dev_lock);
2993 wwn = __srpt_lookup_wwn(name);
2994 spin_unlock(&srpt_dev_lock);
2995
2996 return wwn;
2997}
2998
2999static void srpt_free_srq(struct srpt_device *sdev)
3000{
3001 if (!sdev->srq)
3002 return;
3003
3004 ib_destroy_srq(sdev->srq);
3005 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
3006 sdev->srq_size, sdev->req_buf_cache,
3007 DMA_FROM_DEVICE);
3008 kmem_cache_destroy(sdev->req_buf_cache);
3009 sdev->srq = NULL;
3010}
3011
3012static int srpt_alloc_srq(struct srpt_device *sdev)
3013{
3014 struct ib_srq_init_attr srq_attr = {
3015 .event_handler = srpt_srq_event,
3016 .srq_context = (void *)sdev,
3017 .attr.max_wr = sdev->srq_size,
3018 .attr.max_sge = 1,
3019 .srq_type = IB_SRQT_BASIC,
3020 };
3021 struct ib_device *device = sdev->device;
3022 struct ib_srq *srq;
3023 int i;
3024
3025 WARN_ON_ONCE(sdev->srq);
3026 srq = ib_create_srq(sdev->pd, &srq_attr);
3027 if (IS_ERR(srq)) {
3028 pr_debug("ib_create_srq() failed: %ld\n", PTR_ERR(srq));
3029 return PTR_ERR(srq);
3030 }
3031
3032 pr_debug("create SRQ #wr= %d max_allow=%d dev= %s\n", sdev->srq_size,
3033 sdev->device->attrs.max_srq_wr, dev_name(&device->dev));
3034
3035 sdev->req_buf_cache = kmem_cache_create("srpt-srq-req-buf",
3036 srp_max_req_size, 0, 0, NULL);
3037 if (!sdev->req_buf_cache)
3038 goto free_srq;
3039
3040 sdev->ioctx_ring = (struct srpt_recv_ioctx **)
3041 srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
3042 sizeof(*sdev->ioctx_ring[0]),
3043 sdev->req_buf_cache, 0, DMA_FROM_DEVICE);
3044 if (!sdev->ioctx_ring)
3045 goto free_cache;
3046
3047 sdev->use_srq = true;
3048 sdev->srq = srq;
3049
3050 for (i = 0; i < sdev->srq_size; ++i) {
3051 INIT_LIST_HEAD(&sdev->ioctx_ring[i]->wait_list);
3052 srpt_post_recv(sdev, NULL, sdev->ioctx_ring[i]);
3053 }
3054
3055 return 0;
3056
3057free_cache:
3058 kmem_cache_destroy(sdev->req_buf_cache);
3059
3060free_srq:
3061 ib_destroy_srq(srq);
3062 return -ENOMEM;
3063}
3064
3065static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
3066{
3067 struct ib_device *device = sdev->device;
3068 int ret = 0;
3069
3070 if (!use_srq) {
3071 srpt_free_srq(sdev);
3072 sdev->use_srq = false;
3073 } else if (use_srq && !sdev->srq) {
3074 ret = srpt_alloc_srq(sdev);
3075 }
3076 pr_debug("%s(%s): use_srq = %d; ret = %d\n", __func__,
3077 dev_name(&device->dev), sdev->use_srq, ret);
3078 return ret;
3079}
3080
3081
3082
3083
3084
3085static void srpt_add_one(struct ib_device *device)
3086{
3087 struct srpt_device *sdev;
3088 struct srpt_port *sport;
3089 int i, ret;
3090
3091 pr_debug("device = %p\n", device);
3092
3093 sdev = kzalloc(struct_size(sdev, port, device->phys_port_cnt),
3094 GFP_KERNEL);
3095 if (!sdev)
3096 goto err;
3097
3098 sdev->device = device;
3099 mutex_init(&sdev->sdev_mutex);
3100
3101 sdev->pd = ib_alloc_pd(device, 0);
3102 if (IS_ERR(sdev->pd))
3103 goto free_dev;
3104
3105 sdev->lkey = sdev->pd->local_dma_lkey;
3106
3107 sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
3108
3109 srpt_use_srq(sdev, sdev->port[0].port_attrib.use_srq);
3110
3111 if (!srpt_service_guid)
3112 srpt_service_guid = be64_to_cpu(device->node_guid);
3113
3114 if (rdma_port_get_link_layer(device, 1) == IB_LINK_LAYER_INFINIBAND)
3115 sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
3116 if (IS_ERR(sdev->cm_id)) {
3117 pr_info("ib_create_cm_id() failed: %ld\n",
3118 PTR_ERR(sdev->cm_id));
3119 sdev->cm_id = NULL;
3120 if (!rdma_cm_id)
3121 goto err_ring;
3122 }
3123
3124
3125 pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,pkey=ffff,service_id=%016llx\n",
3126 srpt_service_guid, srpt_service_guid, srpt_service_guid);
3127
3128
3129
3130
3131
3132
3133
3134 ret = sdev->cm_id ?
3135 ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0) :
3136 0;
3137 if (ret < 0) {
3138 pr_err("ib_cm_listen() failed: %d (cm_id state = %d)\n", ret,
3139 sdev->cm_id->state);
3140 goto err_cm;
3141 }
3142
3143 INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
3144 srpt_event_handler);
3145 ib_register_event_handler(&sdev->event_handler);
3146
3147 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3148 sport = &sdev->port[i - 1];
3149 INIT_LIST_HEAD(&sport->nexus_list);
3150 init_waitqueue_head(&sport->ch_releaseQ);
3151 mutex_init(&sport->mutex);
3152 sport->sdev = sdev;
3153 sport->port = i;
3154 sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
3155 sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
3156 sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
3157 sport->port_attrib.use_srq = false;
3158 INIT_WORK(&sport->work, srpt_refresh_port_work);
3159
3160 if (srpt_refresh_port(sport)) {
3161 pr_err("MAD registration failed for %s-%d.\n",
3162 dev_name(&sdev->device->dev), i);
3163 goto err_event;
3164 }
3165 }
3166
3167 spin_lock(&srpt_dev_lock);
3168 list_add_tail(&sdev->list, &srpt_dev_list);
3169 spin_unlock(&srpt_dev_lock);
3170
3171out:
3172 ib_set_client_data(device, &srpt_client, sdev);
3173 pr_debug("added %s.\n", dev_name(&device->dev));
3174 return;
3175
3176err_event:
3177 ib_unregister_event_handler(&sdev->event_handler);
3178err_cm:
3179 if (sdev->cm_id)
3180 ib_destroy_cm_id(sdev->cm_id);
3181err_ring:
3182 srpt_free_srq(sdev);
3183 ib_dealloc_pd(sdev->pd);
3184free_dev:
3185 kfree(sdev);
3186err:
3187 sdev = NULL;
3188 pr_info("%s(%s) failed.\n", __func__, dev_name(&device->dev));
3189 goto out;
3190}
3191
3192
3193
3194
3195
3196
3197static void srpt_remove_one(struct ib_device *device, void *client_data)
3198{
3199 struct srpt_device *sdev = client_data;
3200 int i;
3201
3202 if (!sdev) {
3203 pr_info("%s(%s): nothing to do.\n", __func__,
3204 dev_name(&device->dev));
3205 return;
3206 }
3207
3208 srpt_unregister_mad_agent(sdev);
3209
3210 ib_unregister_event_handler(&sdev->event_handler);
3211
3212
3213 for (i = 0; i < sdev->device->phys_port_cnt; i++)
3214 cancel_work_sync(&sdev->port[i].work);
3215
3216 if (sdev->cm_id)
3217 ib_destroy_cm_id(sdev->cm_id);
3218
3219 ib_set_client_data(device, &srpt_client, NULL);
3220
3221
3222
3223
3224
3225
3226 spin_lock(&srpt_dev_lock);
3227 list_del(&sdev->list);
3228 spin_unlock(&srpt_dev_lock);
3229
3230 for (i = 0; i < sdev->device->phys_port_cnt; i++)
3231 srpt_release_sport(&sdev->port[i]);
3232
3233 srpt_free_srq(sdev);
3234
3235 ib_dealloc_pd(sdev->pd);
3236
3237 kfree(sdev);
3238}
3239
3240static struct ib_client srpt_client = {
3241 .name = DRV_NAME,
3242 .add = srpt_add_one,
3243 .remove = srpt_remove_one
3244};
3245
3246static int srpt_check_true(struct se_portal_group *se_tpg)
3247{
3248 return 1;
3249}
3250
3251static int srpt_check_false(struct se_portal_group *se_tpg)
3252{
3253 return 0;
3254}
3255
3256static struct srpt_port *srpt_tpg_to_sport(struct se_portal_group *tpg)
3257{
3258 return tpg->se_tpg_wwn->priv;
3259}
3260
3261static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
3262{
3263 struct srpt_port *sport = srpt_tpg_to_sport(tpg);
3264
3265 WARN_ON_ONCE(tpg != &sport->port_guid_tpg &&
3266 tpg != &sport->port_gid_tpg);
3267 return tpg == &sport->port_guid_tpg ? sport->port_guid :
3268 sport->port_gid;
3269}
3270
3271static u16 srpt_get_tag(struct se_portal_group *tpg)
3272{
3273 return 1;
3274}
3275
3276static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
3277{
3278 return 1;
3279}
3280
3281static void srpt_release_cmd(struct se_cmd *se_cmd)
3282{
3283 struct srpt_send_ioctx *ioctx = container_of(se_cmd,
3284 struct srpt_send_ioctx, cmd);
3285 struct srpt_rdma_ch *ch = ioctx->ch;
3286 struct srpt_recv_ioctx *recv_ioctx = ioctx->recv_ioctx;
3287
3288 WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
3289 !(ioctx->cmd.transport_state & CMD_T_ABORTED));
3290
3291 if (recv_ioctx) {
3292 WARN_ON_ONCE(!list_empty(&recv_ioctx->wait_list));
3293 ioctx->recv_ioctx = NULL;
3294 srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
3295 }
3296
3297 if (ioctx->n_rw_ctx) {
3298 srpt_free_rw_ctxs(ch, ioctx);
3299 ioctx->n_rw_ctx = 0;
3300 }
3301
3302 target_free_tag(se_cmd->se_sess, se_cmd);
3303}
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313static void srpt_close_session(struct se_session *se_sess)
3314{
3315 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
3316
3317 srpt_disconnect_ch_sync(ch);
3318}
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329static u32 srpt_sess_get_index(struct se_session *se_sess)
3330{
3331 return 0;
3332}
3333
3334static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
3335{
3336}
3337
3338
3339static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
3340{
3341 struct srpt_send_ioctx *ioctx;
3342
3343 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
3344 return ioctx->state;
3345}
3346
3347static int srpt_parse_guid(u64 *guid, const char *name)
3348{
3349 u16 w[4];
3350 int ret = -EINVAL;
3351
3352 if (sscanf(name, "%hx:%hx:%hx:%hx", &w[0], &w[1], &w[2], &w[3]) != 4)
3353 goto out;
3354 *guid = get_unaligned_be64(w);
3355 ret = 0;
3356out:
3357 return ret;
3358}
3359
3360
3361
3362
3363
3364
3365static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
3366{
3367 const char *p;
3368 unsigned len, count, leading_zero_bytes;
3369 int ret;
3370
3371 p = name;
3372 if (strncasecmp(p, "0x", 2) == 0)
3373 p += 2;
3374 ret = -EINVAL;
3375 len = strlen(p);
3376 if (len % 2)
3377 goto out;
3378 count = min(len / 2, 16U);
3379 leading_zero_bytes = 16 - count;
3380 memset(i_port_id, 0, leading_zero_bytes);
3381 ret = hex2bin(i_port_id + leading_zero_bytes, p, count);
3382
3383out:
3384 return ret;
3385}
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
3400{
3401 struct sockaddr_storage sa;
3402 u64 guid;
3403 u8 i_port_id[16];
3404 int ret;
3405
3406 ret = srpt_parse_guid(&guid, name);
3407 if (ret < 0)
3408 ret = srpt_parse_i_port_id(i_port_id, name);
3409 if (ret < 0)
3410 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, name, NULL,
3411 &sa);
3412 if (ret < 0)
3413 pr_err("invalid initiator port ID %s\n", name);
3414 return ret;
3415}
3416
3417static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
3418 char *page)
3419{
3420 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3421 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3422
3423 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
3424}
3425
3426static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
3427 const char *page, size_t count)
3428{
3429 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3430 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3431 unsigned long val;
3432 int ret;
3433
3434 ret = kstrtoul(page, 0, &val);
3435 if (ret < 0) {
3436 pr_err("kstrtoul() failed with ret: %d\n", ret);
3437 return -EINVAL;
3438 }
3439 if (val > MAX_SRPT_RDMA_SIZE) {
3440 pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
3441 MAX_SRPT_RDMA_SIZE);
3442 return -EINVAL;
3443 }
3444 if (val < DEFAULT_MAX_RDMA_SIZE) {
3445 pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
3446 val, DEFAULT_MAX_RDMA_SIZE);
3447 return -EINVAL;
3448 }
3449 sport->port_attrib.srp_max_rdma_size = val;
3450
3451 return count;
3452}
3453
3454static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
3455 char *page)
3456{
3457 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3458 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3459
3460 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
3461}
3462
3463static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
3464 const char *page, size_t count)
3465{
3466 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3467 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3468 unsigned long val;
3469 int ret;
3470
3471 ret = kstrtoul(page, 0, &val);
3472 if (ret < 0) {
3473 pr_err("kstrtoul() failed with ret: %d\n", ret);
3474 return -EINVAL;
3475 }
3476 if (val > MAX_SRPT_RSP_SIZE) {
3477 pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
3478 MAX_SRPT_RSP_SIZE);
3479 return -EINVAL;
3480 }
3481 if (val < MIN_MAX_RSP_SIZE) {
3482 pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
3483 MIN_MAX_RSP_SIZE);
3484 return -EINVAL;
3485 }
3486 sport->port_attrib.srp_max_rsp_size = val;
3487
3488 return count;
3489}
3490
3491static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
3492 char *page)
3493{
3494 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3495 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3496
3497 return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
3498}
3499
3500static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
3501 const char *page, size_t count)
3502{
3503 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3504 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3505 unsigned long val;
3506 int ret;
3507
3508 ret = kstrtoul(page, 0, &val);
3509 if (ret < 0) {
3510 pr_err("kstrtoul() failed with ret: %d\n", ret);
3511 return -EINVAL;
3512 }
3513 if (val > MAX_SRPT_SRQ_SIZE) {
3514 pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
3515 MAX_SRPT_SRQ_SIZE);
3516 return -EINVAL;
3517 }
3518 if (val < MIN_SRPT_SRQ_SIZE) {
3519 pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
3520 MIN_SRPT_SRQ_SIZE);
3521 return -EINVAL;
3522 }
3523 sport->port_attrib.srp_sq_size = val;
3524
3525 return count;
3526}
3527
3528static ssize_t srpt_tpg_attrib_use_srq_show(struct config_item *item,
3529 char *page)
3530{
3531 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3532 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3533
3534 return sprintf(page, "%d\n", sport->port_attrib.use_srq);
3535}
3536
3537static ssize_t srpt_tpg_attrib_use_srq_store(struct config_item *item,
3538 const char *page, size_t count)
3539{
3540 struct se_portal_group *se_tpg = attrib_to_tpg(item);
3541 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3542 struct srpt_device *sdev = sport->sdev;
3543 unsigned long val;
3544 bool enabled;
3545 int ret;
3546
3547 ret = kstrtoul(page, 0, &val);
3548 if (ret < 0)
3549 return ret;
3550 if (val != !!val)
3551 return -EINVAL;
3552
3553 ret = mutex_lock_interruptible(&sdev->sdev_mutex);
3554 if (ret < 0)
3555 return ret;
3556 ret = mutex_lock_interruptible(&sport->mutex);
3557 if (ret < 0)
3558 goto unlock_sdev;
3559 enabled = sport->enabled;
3560
3561 srpt_set_enabled(sport, false);
3562 sport->port_attrib.use_srq = val;
3563 srpt_use_srq(sdev, sport->port_attrib.use_srq);
3564 srpt_set_enabled(sport, enabled);
3565 ret = count;
3566 mutex_unlock(&sport->mutex);
3567unlock_sdev:
3568 mutex_unlock(&sdev->sdev_mutex);
3569
3570 return ret;
3571}
3572
3573CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size);
3574CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size);
3575CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size);
3576CONFIGFS_ATTR(srpt_tpg_attrib_, use_srq);
3577
3578static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
3579 &srpt_tpg_attrib_attr_srp_max_rdma_size,
3580 &srpt_tpg_attrib_attr_srp_max_rsp_size,
3581 &srpt_tpg_attrib_attr_srp_sq_size,
3582 &srpt_tpg_attrib_attr_use_srq,
3583 NULL,
3584};
3585
3586static struct rdma_cm_id *srpt_create_rdma_id(struct sockaddr *listen_addr)
3587{
3588 struct rdma_cm_id *rdma_cm_id;
3589 int ret;
3590
3591 rdma_cm_id = rdma_create_id(&init_net, srpt_rdma_cm_handler,
3592 NULL, RDMA_PS_TCP, IB_QPT_RC);
3593 if (IS_ERR(rdma_cm_id)) {
3594 pr_err("RDMA/CM ID creation failed: %ld\n",
3595 PTR_ERR(rdma_cm_id));
3596 goto out;
3597 }
3598
3599 ret = rdma_bind_addr(rdma_cm_id, listen_addr);
3600 if (ret) {
3601 char addr_str[64];
3602
3603 snprintf(addr_str, sizeof(addr_str), "%pISp", listen_addr);
3604 pr_err("Binding RDMA/CM ID to address %s failed: %d\n",
3605 addr_str, ret);
3606 rdma_destroy_id(rdma_cm_id);
3607 rdma_cm_id = ERR_PTR(ret);
3608 goto out;
3609 }
3610
3611 ret = rdma_listen(rdma_cm_id, 128);
3612 if (ret) {
3613 pr_err("rdma_listen() failed: %d\n", ret);
3614 rdma_destroy_id(rdma_cm_id);
3615 rdma_cm_id = ERR_PTR(ret);
3616 }
3617
3618out:
3619 return rdma_cm_id;
3620}
3621
3622static ssize_t srpt_rdma_cm_port_show(struct config_item *item, char *page)
3623{
3624 return sprintf(page, "%d\n", rdma_cm_port);
3625}
3626
3627static ssize_t srpt_rdma_cm_port_store(struct config_item *item,
3628 const char *page, size_t count)
3629{
3630 struct sockaddr_in addr4 = { .sin_family = AF_INET };
3631 struct sockaddr_in6 addr6 = { .sin6_family = AF_INET6 };
3632 struct rdma_cm_id *new_id = NULL;
3633 u16 val;
3634 int ret;
3635
3636 ret = kstrtou16(page, 0, &val);
3637 if (ret < 0)
3638 return ret;
3639 ret = count;
3640 if (rdma_cm_port == val)
3641 goto out;
3642
3643 if (val) {
3644 addr6.sin6_port = cpu_to_be16(val);
3645 new_id = srpt_create_rdma_id((struct sockaddr *)&addr6);
3646 if (IS_ERR(new_id)) {
3647 addr4.sin_port = cpu_to_be16(val);
3648 new_id = srpt_create_rdma_id((struct sockaddr *)&addr4);
3649 if (IS_ERR(new_id)) {
3650 ret = PTR_ERR(new_id);
3651 goto out;
3652 }
3653 }
3654 }
3655
3656 mutex_lock(&rdma_cm_mutex);
3657 rdma_cm_port = val;
3658 swap(rdma_cm_id, new_id);
3659 mutex_unlock(&rdma_cm_mutex);
3660
3661 if (new_id)
3662 rdma_destroy_id(new_id);
3663 ret = count;
3664out:
3665 return ret;
3666}
3667
3668CONFIGFS_ATTR(srpt_, rdma_cm_port);
3669
3670static struct configfs_attribute *srpt_da_attrs[] = {
3671 &srpt_attr_rdma_cm_port,
3672 NULL,
3673};
3674
3675static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
3676{
3677 struct se_portal_group *se_tpg = to_tpg(item);
3678 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3679
3680 return snprintf(page, PAGE_SIZE, "%d\n", sport->enabled);
3681}
3682
3683static ssize_t srpt_tpg_enable_store(struct config_item *item,
3684 const char *page, size_t count)
3685{
3686 struct se_portal_group *se_tpg = to_tpg(item);
3687 struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3688 unsigned long tmp;
3689 int ret;
3690
3691 ret = kstrtoul(page, 0, &tmp);
3692 if (ret < 0) {
3693 pr_err("Unable to extract srpt_tpg_store_enable\n");
3694 return -EINVAL;
3695 }
3696
3697 if ((tmp != 0) && (tmp != 1)) {
3698 pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
3699 return -EINVAL;
3700 }
3701
3702 mutex_lock(&sport->mutex);
3703 srpt_set_enabled(sport, tmp);
3704 mutex_unlock(&sport->mutex);
3705
3706 return count;
3707}
3708
3709CONFIGFS_ATTR(srpt_tpg_, enable);
3710
3711static struct configfs_attribute *srpt_tpg_attrs[] = {
3712 &srpt_tpg_attr_enable,
3713 NULL,
3714};
3715
3716
3717
3718
3719
3720
3721static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
3722 const char *name)
3723{
3724 struct srpt_port *sport = wwn->priv;
3725 struct se_portal_group *tpg;
3726 int res;
3727
3728 WARN_ON_ONCE(wwn != &sport->port_guid_wwn &&
3729 wwn != &sport->port_gid_wwn);
3730 tpg = wwn == &sport->port_guid_wwn ? &sport->port_guid_tpg :
3731 &sport->port_gid_tpg;
3732 res = core_tpg_register(wwn, tpg, SCSI_PROTOCOL_SRP);
3733 if (res)
3734 return ERR_PTR(res);
3735
3736 return tpg;
3737}
3738
3739
3740
3741
3742
3743static void srpt_drop_tpg(struct se_portal_group *tpg)
3744{
3745 struct srpt_port *sport = srpt_tpg_to_sport(tpg);
3746
3747 sport->enabled = false;
3748 core_tpg_deregister(tpg);
3749}
3750
3751
3752
3753
3754
3755
3756
3757static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3758 struct config_group *group,
3759 const char *name)
3760{
3761 return srpt_lookup_wwn(name) ? : ERR_PTR(-EINVAL);
3762}
3763
3764
3765
3766
3767
3768static void srpt_drop_tport(struct se_wwn *wwn)
3769{
3770}
3771
3772static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
3773{
3774 return scnprintf(buf, PAGE_SIZE, "\n");
3775}
3776
3777CONFIGFS_ATTR_RO(srpt_wwn_, version);
3778
3779static struct configfs_attribute *srpt_wwn_attrs[] = {
3780 &srpt_wwn_attr_version,
3781 NULL,
3782};
3783
3784static const struct target_core_fabric_ops srpt_template = {
3785 .module = THIS_MODULE,
3786 .fabric_name = "srpt",
3787 .tpg_get_wwn = srpt_get_fabric_wwn,
3788 .tpg_get_tag = srpt_get_tag,
3789 .tpg_check_demo_mode = srpt_check_false,
3790 .tpg_check_demo_mode_cache = srpt_check_true,
3791 .tpg_check_demo_mode_write_protect = srpt_check_true,
3792 .tpg_check_prod_mode_write_protect = srpt_check_false,
3793 .tpg_get_inst_index = srpt_tpg_get_inst_index,
3794 .release_cmd = srpt_release_cmd,
3795 .check_stop_free = srpt_check_stop_free,
3796 .close_session = srpt_close_session,
3797 .sess_get_index = srpt_sess_get_index,
3798 .sess_get_initiator_sid = NULL,
3799 .write_pending = srpt_write_pending,
3800 .set_default_node_attributes = srpt_set_default_node_attrs,
3801 .get_cmd_state = srpt_get_tcm_cmd_state,
3802 .queue_data_in = srpt_queue_data_in,
3803 .queue_status = srpt_queue_status,
3804 .queue_tm_rsp = srpt_queue_tm_rsp,
3805 .aborted_task = srpt_aborted_task,
3806
3807
3808
3809
3810 .fabric_make_wwn = srpt_make_tport,
3811 .fabric_drop_wwn = srpt_drop_tport,
3812 .fabric_make_tpg = srpt_make_tpg,
3813 .fabric_drop_tpg = srpt_drop_tpg,
3814 .fabric_init_nodeacl = srpt_init_nodeacl,
3815
3816 .tfc_discovery_attrs = srpt_da_attrs,
3817 .tfc_wwn_attrs = srpt_wwn_attrs,
3818 .tfc_tpg_base_attrs = srpt_tpg_attrs,
3819 .tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs,
3820};
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830static int __init srpt_init_module(void)
3831{
3832 int ret;
3833
3834 ret = -EINVAL;
3835 if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3836 pr_err("invalid value %d for kernel module parameter srp_max_req_size -- must be at least %d.\n",
3837 srp_max_req_size, MIN_MAX_REQ_SIZE);
3838 goto out;
3839 }
3840
3841 if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3842 || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3843 pr_err("invalid value %d for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n",
3844 srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3845 goto out;
3846 }
3847
3848 ret = target_register_template(&srpt_template);
3849 if (ret)
3850 goto out;
3851
3852 ret = ib_register_client(&srpt_client);
3853 if (ret) {
3854 pr_err("couldn't register IB client\n");
3855 goto out_unregister_target;
3856 }
3857
3858 return 0;
3859
3860out_unregister_target:
3861 target_unregister_template(&srpt_template);
3862out:
3863 return ret;
3864}
3865
3866static void __exit srpt_cleanup_module(void)
3867{
3868 if (rdma_cm_id)
3869 rdma_destroy_id(rdma_cm_id);
3870 ib_unregister_client(&srpt_client);
3871 target_unregister_template(&srpt_template);
3872}
3873
3874module_init(srpt_init_module);
3875module_exit(srpt_cleanup_module);
3876