1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#ifndef __ISCSI_ISER_H__
42#define __ISCSI_ISER_H__
43
44#include <linux/types.h>
45#include <linux/net.h>
46#include <linux/printk.h>
47#include <scsi/libiscsi.h>
48#include <scsi/scsi_transport_iscsi.h>
49#include <scsi/scsi_cmnd.h>
50#include <scsi/scsi_device.h>
51#include <scsi/iser.h>
52
53#include <linux/interrupt.h>
54#include <linux/wait.h>
55#include <linux/sched.h>
56#include <linux/list.h>
57#include <linux/slab.h>
58#include <linux/dma-mapping.h>
59#include <linux/mutex.h>
60#include <linux/mempool.h>
61#include <linux/uio.h>
62
63#include <linux/socket.h>
64#include <linux/in.h>
65#include <linux/in6.h>
66
67#include <rdma/ib_verbs.h>
68#include <rdma/ib_fmr_pool.h>
69#include <rdma/rdma_cm.h>
70
71#define DRV_NAME "iser"
72#define PFX DRV_NAME ": "
73#define DRV_VER "1.6"
74
75#define iser_dbg(fmt, arg...) \
76 do { \
77 if (unlikely(iser_debug_level > 2)) \
78 printk(KERN_DEBUG PFX "%s: " fmt,\
79 __func__ , ## arg); \
80 } while (0)
81
82#define iser_warn(fmt, arg...) \
83 do { \
84 if (unlikely(iser_debug_level > 0)) \
85 pr_warn(PFX "%s: " fmt, \
86 __func__ , ## arg); \
87 } while (0)
88
89#define iser_info(fmt, arg...) \
90 do { \
91 if (unlikely(iser_debug_level > 1)) \
92 pr_info(PFX "%s: " fmt, \
93 __func__ , ## arg); \
94 } while (0)
95
96#define iser_err(fmt, arg...) \
97 pr_err(PFX "%s: " fmt, __func__ , ## arg)
98
99#define SHIFT_4K 12
100#define SIZE_4K (1ULL << SHIFT_4K)
101#define MASK_4K (~(SIZE_4K-1))
102
103
104#define ISER_DEF_MAX_SECTORS 1024
105#define ISCSI_ISER_DEF_SG_TABLESIZE ((ISER_DEF_MAX_SECTORS * 512) >> SHIFT_4K)
106
107#define ISCSI_ISER_MAX_SG_TABLESIZE ((16384 * 512) >> SHIFT_4K)
108
109#define ISER_DEF_XMIT_CMDS_DEFAULT 512
110#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
111 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
112#else
113 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
114#endif
115#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
116
117
118
119#define ISER_MAX_RX_MISC_PDUS 4
120
121#define ISER_MAX_TX_MISC_PDUS 6
122
123
124#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
125
126#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
127
128
129
130
131
132
133
134#define ISER_INFLIGHT_DATAOUTS 8
135
136#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
137 (1 + ISER_INFLIGHT_DATAOUTS) + \
138 ISER_MAX_TX_MISC_PDUS + \
139 ISER_MAX_RX_MISC_PDUS)
140
141
142#define ISER_MAX_REG_WR_PER_CMD 5
143
144
145#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
146 (1 + ISER_MAX_REG_WR_PER_CMD) + \
147 ISER_MAX_TX_MISC_PDUS + \
148 ISER_MAX_RX_MISC_PDUS)
149
150#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
151 - ISER_MAX_TX_MISC_PDUS \
152 - ISER_MAX_RX_MISC_PDUS) / \
153 (1 + ISER_INFLIGHT_DATAOUTS))
154
155#define ISER_SIGNAL_CMD_COUNT 32
156
157
158#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
159
160#define ISER_RECV_DATA_SEG_LEN 128
161#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
162#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
163
164
165#define ISER_OBJECT_NAME_SIZE 64
166
167enum iser_conn_state {
168 ISER_CONN_INIT,
169 ISER_CONN_PENDING,
170 ISER_CONN_UP,
171 ISER_CONN_TERMINATING,
172 ISER_CONN_DOWN,
173 ISER_CONN_STATES_NUM
174};
175
176enum iser_task_status {
177 ISER_TASK_STATUS_INIT = 0,
178 ISER_TASK_STATUS_STARTED,
179 ISER_TASK_STATUS_COMPLETED
180};
181
182enum iser_data_dir {
183 ISER_DIR_IN = 0,
184 ISER_DIR_OUT,
185 ISER_DIRS_NUM
186};
187
188
189
190
191
192
193
194
195
196struct iser_data_buf {
197 struct scatterlist *sg;
198 int size;
199 unsigned long data_len;
200 unsigned int dma_nents;
201};
202
203
204struct iser_device;
205struct iscsi_iser_task;
206struct iscsi_endpoint;
207struct iser_reg_resources;
208
209
210
211
212
213
214
215
216struct iser_mem_reg {
217 struct ib_sge sge;
218 u32 rkey;
219 void *mem_h;
220};
221
222enum iser_desc_type {
223 ISCSI_TX_CONTROL ,
224 ISCSI_TX_SCSI_COMMAND,
225 ISCSI_TX_DATAOUT
226};
227
228
229
230
231
232
233
234#define ISER_MAX_WRS 7
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254struct iser_tx_desc {
255 struct iser_ctrl iser_header;
256 struct iscsi_hdr iscsi_header;
257 enum iser_desc_type type;
258 u64 dma_addr;
259 struct ib_sge tx_sg[2];
260 int num_sge;
261 struct ib_cqe cqe;
262 bool mapped;
263 u8 wr_idx;
264 union iser_wr {
265 struct ib_send_wr send;
266 struct ib_reg_wr fast_reg;
267 struct ib_sig_handover_wr sig;
268 } wrs[ISER_MAX_WRS];
269 struct iser_mem_reg data_reg;
270 struct iser_mem_reg prot_reg;
271 struct ib_sig_attrs sig_attrs;
272};
273
274#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
275 sizeof(u64) + sizeof(struct ib_sge) + \
276 sizeof(struct ib_cqe)))
277
278
279
280
281
282
283
284
285
286
287struct iser_rx_desc {
288 struct iser_ctrl iser_header;
289 struct iscsi_hdr iscsi_header;
290 char data[ISER_RECV_DATA_SEG_LEN];
291 u64 dma_addr;
292 struct ib_sge rx_sg;
293 struct ib_cqe cqe;
294 char pad[ISER_RX_PAD_SIZE];
295} __packed;
296
297
298
299
300
301
302
303
304
305
306
307struct iser_login_desc {
308 void *req;
309 void *rsp;
310 u64 req_dma;
311 u64 rsp_dma;
312 struct ib_sge sge;
313 struct ib_cqe cqe;
314} __attribute__((packed));
315
316struct iser_conn;
317struct ib_conn;
318struct iscsi_iser_task;
319
320
321
322
323
324
325
326
327struct iser_comp {
328 struct ib_cq *cq;
329 int active_qps;
330};
331
332
333
334
335
336
337
338
339
340
341
342
343struct iser_reg_ops {
344 int (*alloc_reg_res)(struct ib_conn *ib_conn,
345 unsigned cmds_max,
346 unsigned int size);
347 void (*free_reg_res)(struct ib_conn *ib_conn);
348 int (*reg_mem)(struct iscsi_iser_task *iser_task,
349 struct iser_data_buf *mem,
350 struct iser_reg_resources *rsc,
351 struct iser_mem_reg *reg);
352 void (*unreg_mem)(struct iscsi_iser_task *iser_task,
353 enum iser_data_dir cmd_dir);
354 struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
355 void (*reg_desc_put)(struct ib_conn *ib_conn,
356 struct iser_fr_desc *desc);
357};
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374struct iser_device {
375 struct ib_device *ib_device;
376 struct ib_pd *pd;
377 struct ib_mr *mr;
378 struct ib_event_handler event_handler;
379 struct list_head ig_list;
380 int refcount;
381 int comps_used;
382 struct iser_comp *comps;
383 const struct iser_reg_ops *reg_ops;
384 bool remote_inv_sup;
385};
386
387#define ISER_CHECK_GUARD 0xc0
388#define ISER_CHECK_REFTAG 0x0f
389#define ISER_CHECK_APPTAG 0x30
390
391
392
393
394
395
396
397
398
399struct iser_reg_resources {
400 union {
401 struct ib_mr *mr;
402 struct ib_fmr_pool *fmr_pool;
403 };
404 struct iser_page_vec *page_vec;
405 u8 mr_valid:1;
406};
407
408
409
410
411
412
413
414
415
416struct iser_pi_context {
417 struct iser_reg_resources rsc;
418 struct ib_mr *sig_mr;
419 u8 sig_mr_valid:1;
420 u8 sig_protected:1;
421};
422
423
424
425
426
427
428
429
430struct iser_fr_desc {
431 struct list_head list;
432 struct iser_reg_resources rsc;
433 struct iser_pi_context *pi_ctx;
434};
435
436
437
438
439
440
441
442
443struct iser_fr_pool {
444 struct list_head list;
445 spinlock_t lock;
446 int size;
447};
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462struct ib_conn {
463 struct rdma_cm_id *cma_id;
464 struct ib_qp *qp;
465 int post_recv_buf_count;
466 u8 sig_count;
467 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
468 struct iser_device *device;
469 struct iser_comp *comp;
470 struct iser_fr_pool fr_pool;
471 bool pi_support;
472 struct ib_cqe reg_cqe;
473};
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502struct iser_conn {
503 struct ib_conn ib_conn;
504 struct iscsi_conn *iscsi_conn;
505 struct iscsi_endpoint *ep;
506 enum iser_conn_state state;
507 unsigned qp_max_recv_dtos;
508 unsigned qp_max_recv_dtos_mask;
509 unsigned min_posted_rx;
510 u16 max_cmds;
511 char name[ISER_OBJECT_NAME_SIZE];
512 struct work_struct release_work;
513 struct mutex state_mutex;
514 struct completion stop_completion;
515 struct completion ib_completion;
516 struct completion up_completion;
517 struct list_head conn_list;
518 struct iser_login_desc login_desc;
519 unsigned int rx_desc_head;
520 struct iser_rx_desc *rx_descs;
521 u32 num_rx_descs;
522 unsigned short scsi_sg_tablesize;
523 unsigned int scsi_max_sectors;
524 bool snd_w_inv;
525};
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540struct iscsi_iser_task {
541 struct iser_tx_desc desc;
542 struct iser_conn *iser_conn;
543 enum iser_task_status status;
544 struct scsi_cmnd *sc;
545 int command_sent;
546 int dir[ISER_DIRS_NUM];
547 struct iser_mem_reg rdma_reg[ISER_DIRS_NUM];
548 struct iser_data_buf data[ISER_DIRS_NUM];
549 struct iser_data_buf prot[ISER_DIRS_NUM];
550};
551
552struct iser_page_vec {
553 u64 *pages;
554 int npages;
555 struct ib_mr fake_mr;
556};
557
558
559
560
561
562
563
564
565
566
567struct iser_global {
568 struct mutex device_list_mutex;
569 struct list_head device_list;
570 struct mutex connlist_mutex;
571 struct list_head connlist;
572 struct kmem_cache *desc_cache;
573};
574
575extern struct iser_global ig;
576extern int iser_debug_level;
577extern bool iser_pi_enable;
578extern int iser_pi_guard;
579extern unsigned int iser_max_sectors;
580extern bool iser_always_reg;
581
582int iser_assign_reg_ops(struct iser_device *device);
583
584int iser_send_control(struct iscsi_conn *conn,
585 struct iscsi_task *task);
586
587int iser_send_command(struct iscsi_conn *conn,
588 struct iscsi_task *task);
589
590int iser_send_data_out(struct iscsi_conn *conn,
591 struct iscsi_task *task,
592 struct iscsi_data *hdr);
593
594void iscsi_iser_recv(struct iscsi_conn *conn,
595 struct iscsi_hdr *hdr,
596 char *rx_data,
597 int rx_data_len);
598
599void iser_conn_init(struct iser_conn *iser_conn);
600
601void iser_conn_release(struct iser_conn *iser_conn);
602
603int iser_conn_terminate(struct iser_conn *iser_conn);
604
605void iser_release_work(struct work_struct *work);
606
607void iser_err_comp(struct ib_wc *wc, const char *type);
608void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc);
609void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc);
610void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc);
611void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc);
612void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc);
613void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc);
614
615void iser_task_rdma_init(struct iscsi_iser_task *task);
616
617void iser_task_rdma_finalize(struct iscsi_iser_task *task);
618
619void iser_free_rx_descriptors(struct iser_conn *iser_conn);
620
621void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
622 struct iser_data_buf *mem,
623 enum iser_data_dir cmd_dir);
624
625int iser_reg_rdma_mem(struct iscsi_iser_task *task,
626 enum iser_data_dir dir,
627 bool all_imm);
628void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
629 enum iser_data_dir dir);
630
631int iser_connect(struct iser_conn *iser_conn,
632 struct sockaddr *src_addr,
633 struct sockaddr *dst_addr,
634 int non_blocking);
635
636void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
637 enum iser_data_dir cmd_dir);
638void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
639 enum iser_data_dir cmd_dir);
640
641int iser_post_recvl(struct iser_conn *iser_conn);
642int iser_post_recvm(struct iser_conn *iser_conn, int count);
643int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
644 bool signal);
645
646int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
647 struct iser_data_buf *data,
648 enum iser_data_dir iser_dir,
649 enum dma_data_direction dma_dir);
650
651void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
652 struct iser_data_buf *data,
653 enum dma_data_direction dir);
654
655int iser_initialize_task_headers(struct iscsi_task *task,
656 struct iser_tx_desc *tx_desc);
657int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
658 struct iscsi_session *session);
659int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
660 unsigned cmds_max,
661 unsigned int size);
662void iser_free_fmr_pool(struct ib_conn *ib_conn);
663int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
664 unsigned cmds_max,
665 unsigned int size);
666void iser_free_fastreg_pool(struct ib_conn *ib_conn);
667u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
668 enum iser_data_dir cmd_dir, sector_t *sector);
669struct iser_fr_desc *
670iser_reg_desc_get_fr(struct ib_conn *ib_conn);
671void
672iser_reg_desc_put_fr(struct ib_conn *ib_conn,
673 struct iser_fr_desc *desc);
674struct iser_fr_desc *
675iser_reg_desc_get_fmr(struct ib_conn *ib_conn);
676void
677iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
678 struct iser_fr_desc *desc);
679
680static inline struct ib_send_wr *
681iser_tx_next_wr(struct iser_tx_desc *tx_desc)
682{
683 struct ib_send_wr *cur_wr = &tx_desc->wrs[tx_desc->wr_idx].send;
684 struct ib_send_wr *last_wr;
685
686 if (tx_desc->wr_idx) {
687 last_wr = &tx_desc->wrs[tx_desc->wr_idx - 1].send;
688 last_wr->next = cur_wr;
689 }
690 tx_desc->wr_idx++;
691
692 return cur_wr;
693}
694
695static inline struct iser_conn *
696to_iser_conn(struct ib_conn *ib_conn)
697{
698 return container_of(ib_conn, struct iser_conn, ib_conn);
699}
700
701static inline struct iser_rx_desc *
702iser_rx(struct ib_cqe *cqe)
703{
704 return container_of(cqe, struct iser_rx_desc, cqe);
705}
706
707static inline struct iser_tx_desc *
708iser_tx(struct ib_cqe *cqe)
709{
710 return container_of(cqe, struct iser_tx_desc, cqe);
711}
712
713static inline struct iser_login_desc *
714iser_login(struct ib_cqe *cqe)
715{
716 return container_of(cqe, struct iser_login_desc, cqe);
717}
718
719#endif
720