1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#ifndef __ISCSI_ISER_H__
42#define __ISCSI_ISER_H__
43
44#include <linux/types.h>
45#include <linux/net.h>
46#include <linux/printk.h>
47#include <scsi/libiscsi.h>
48#include <scsi/scsi_transport_iscsi.h>
49#include <scsi/scsi_cmnd.h>
50#include <scsi/scsi_device.h>
51#include <scsi/iser.h>
52
53#include <linux/interrupt.h>
54#include <linux/wait.h>
55#include <linux/sched.h>
56#include <linux/list.h>
57#include <linux/slab.h>
58#include <linux/dma-mapping.h>
59#include <linux/mutex.h>
60#include <linux/mempool.h>
61#include <linux/uio.h>
62
63#include <linux/socket.h>
64#include <linux/in.h>
65#include <linux/in6.h>
66
67#include <rdma/ib_verbs.h>
68#include <rdma/ib_fmr_pool.h>
69#include <rdma/rdma_cm.h>
70
71#define DRV_NAME "iser"
72#define PFX DRV_NAME ": "
73#define DRV_VER "1.6"
74
75#define iser_dbg(fmt, arg...) \
76 do { \
77 if (unlikely(iser_debug_level > 2)) \
78 printk(KERN_DEBUG PFX "%s: " fmt,\
79 __func__ , ## arg); \
80 } while (0)
81
82#define iser_warn(fmt, arg...) \
83 do { \
84 if (unlikely(iser_debug_level > 0)) \
85 pr_warn(PFX "%s: " fmt, \
86 __func__ , ## arg); \
87 } while (0)
88
89#define iser_info(fmt, arg...) \
90 do { \
91 if (unlikely(iser_debug_level > 1)) \
92 pr_info(PFX "%s: " fmt, \
93 __func__ , ## arg); \
94 } while (0)
95
96#define iser_err(fmt, arg...) \
97 pr_err(PFX "%s: " fmt, __func__ , ## arg)
98
99#define SHIFT_4K 12
100#define SIZE_4K (1ULL << SHIFT_4K)
101#define MASK_4K (~(SIZE_4K-1))
102
103
104#define ISER_DEF_MAX_SECTORS 1024
105#define ISCSI_ISER_DEF_SG_TABLESIZE ((ISER_DEF_MAX_SECTORS * 512) >> SHIFT_4K)
106
107#define ISCSI_ISER_MAX_SG_TABLESIZE ((16384 * 512) >> SHIFT_4K)
108
109#define ISER_DEF_XMIT_CMDS_DEFAULT 512
110#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
111 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
112#else
113 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
114#endif
115#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
116
117
118
119#define ISER_MAX_RX_MISC_PDUS 4
120
121#define ISER_MAX_TX_MISC_PDUS 6
122
123
124#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
125
126#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
127
128
129
130
131
132
133
134#define ISER_INFLIGHT_DATAOUTS 8
135
136#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
137 (1 + ISER_INFLIGHT_DATAOUTS) + \
138 ISER_MAX_TX_MISC_PDUS + \
139 ISER_MAX_RX_MISC_PDUS)
140
141
142#define ISER_MAX_REG_WR_PER_CMD 5
143
144
145#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
146 (1 + ISER_MAX_REG_WR_PER_CMD) + \
147 ISER_MAX_TX_MISC_PDUS + \
148 ISER_MAX_RX_MISC_PDUS)
149
150#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
151 - ISER_MAX_TX_MISC_PDUS \
152 - ISER_MAX_RX_MISC_PDUS) / \
153 (1 + ISER_INFLIGHT_DATAOUTS))
154
155#define ISER_SIGNAL_CMD_COUNT 32
156
157
158#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
159
160#define ISER_RECV_DATA_SEG_LEN 128
161#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
162#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
163
164
165#define ISER_OBJECT_NAME_SIZE 64
166
167enum iser_conn_state {
168 ISER_CONN_INIT,
169 ISER_CONN_PENDING,
170 ISER_CONN_UP,
171 ISER_CONN_TERMINATING,
172 ISER_CONN_DOWN,
173 ISER_CONN_STATES_NUM
174};
175
176enum iser_task_status {
177 ISER_TASK_STATUS_INIT = 0,
178 ISER_TASK_STATUS_STARTED,
179 ISER_TASK_STATUS_COMPLETED
180};
181
182enum iser_data_dir {
183 ISER_DIR_IN = 0,
184 ISER_DIR_OUT,
185 ISER_DIRS_NUM
186};
187
188
189
190
191
192
193
194
195
196struct iser_data_buf {
197 struct scatterlist *sg;
198 int size;
199 unsigned long data_len;
200 unsigned int dma_nents;
201};
202
203
204struct iser_device;
205struct iscsi_iser_task;
206struct iscsi_endpoint;
207struct iser_reg_resources;
208
209
210
211
212
213
214
215
216struct iser_mem_reg {
217 struct ib_sge sge;
218 u32 rkey;
219 void *mem_h;
220};
221
222enum iser_desc_type {
223 ISCSI_TX_CONTROL ,
224 ISCSI_TX_SCSI_COMMAND,
225 ISCSI_TX_DATAOUT
226};
227
228
229
230
231
232
233
234#define ISER_MAX_WRS 7
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254struct iser_tx_desc {
255 struct iser_ctrl iser_header;
256 struct iscsi_hdr iscsi_header;
257 enum iser_desc_type type;
258 u64 dma_addr;
259 struct ib_sge tx_sg[2];
260 int num_sge;
261 struct ib_cqe cqe;
262 bool mapped;
263 u8 wr_idx;
264 union iser_wr {
265 struct ib_send_wr send;
266 struct ib_reg_wr fast_reg;
267 struct ib_sig_handover_wr sig;
268 } wrs[ISER_MAX_WRS];
269 struct iser_mem_reg data_reg;
270 struct iser_mem_reg prot_reg;
271 struct ib_sig_attrs sig_attrs;
272};
273
274#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
275 sizeof(u64) + sizeof(struct ib_sge) + \
276 sizeof(struct ib_cqe)))
277
278
279
280
281
282
283
284
285
286
287struct iser_rx_desc {
288 struct iser_ctrl iser_header;
289 struct iscsi_hdr iscsi_header;
290 char data[ISER_RECV_DATA_SEG_LEN];
291 u64 dma_addr;
292 struct ib_sge rx_sg;
293 struct ib_cqe cqe;
294 char pad[ISER_RX_PAD_SIZE];
295} __packed;
296
297
298
299
300
301
302
303
304
305
306
307struct iser_login_desc {
308 void *req;
309 void *rsp;
310 u64 req_dma;
311 u64 rsp_dma;
312 struct ib_sge sge;
313 struct ib_cqe cqe;
314} __attribute__((packed));
315
316struct iser_conn;
317struct ib_conn;
318struct iscsi_iser_task;
319
320
321
322
323
324
325
326
327struct iser_comp {
328 struct ib_cq *cq;
329 int active_qps;
330};
331
332
333
334
335
336
337
338
339
340
341
342
343struct iser_reg_ops {
344 int (*alloc_reg_res)(struct ib_conn *ib_conn,
345 unsigned cmds_max,
346 unsigned int size);
347 void (*free_reg_res)(struct ib_conn *ib_conn);
348 int (*reg_mem)(struct iscsi_iser_task *iser_task,
349 struct iser_data_buf *mem,
350 struct iser_reg_resources *rsc,
351 struct iser_mem_reg *reg);
352 void (*unreg_mem)(struct iscsi_iser_task *iser_task,
353 enum iser_data_dir cmd_dir);
354 struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
355 void (*reg_desc_put)(struct ib_conn *ib_conn,
356 struct iser_fr_desc *desc);
357};
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374struct iser_device {
375 struct ib_device *ib_device;
376 struct ib_pd *pd;
377 struct ib_event_handler event_handler;
378 struct list_head ig_list;
379 int refcount;
380 int comps_used;
381 struct iser_comp *comps;
382 const struct iser_reg_ops *reg_ops;
383 bool remote_inv_sup;
384};
385
386#define ISER_CHECK_GUARD 0xc0
387#define ISER_CHECK_REFTAG 0x0f
388#define ISER_CHECK_APPTAG 0x30
389
390
391
392
393
394
395
396
397
398struct iser_reg_resources {
399 union {
400 struct ib_mr *mr;
401 struct ib_fmr_pool *fmr_pool;
402 };
403 struct iser_page_vec *page_vec;
404 u8 mr_valid:1;
405};
406
407
408
409
410
411
412
413
414
415struct iser_pi_context {
416 struct iser_reg_resources rsc;
417 struct ib_mr *sig_mr;
418 u8 sig_mr_valid:1;
419 u8 sig_protected:1;
420};
421
422
423
424
425
426
427
428
429struct iser_fr_desc {
430 struct list_head list;
431 struct iser_reg_resources rsc;
432 struct iser_pi_context *pi_ctx;
433};
434
435
436
437
438
439
440
441
442struct iser_fr_pool {
443 struct list_head list;
444 spinlock_t lock;
445 int size;
446};
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461struct ib_conn {
462 struct rdma_cm_id *cma_id;
463 struct ib_qp *qp;
464 int post_recv_buf_count;
465 u8 sig_count;
466 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
467 struct iser_device *device;
468 struct iser_comp *comp;
469 struct iser_fr_pool fr_pool;
470 bool pi_support;
471 struct ib_cqe reg_cqe;
472};
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501struct iser_conn {
502 struct ib_conn ib_conn;
503 struct iscsi_conn *iscsi_conn;
504 struct iscsi_endpoint *ep;
505 enum iser_conn_state state;
506 unsigned qp_max_recv_dtos;
507 unsigned qp_max_recv_dtos_mask;
508 unsigned min_posted_rx;
509 u16 max_cmds;
510 char name[ISER_OBJECT_NAME_SIZE];
511 struct work_struct release_work;
512 struct mutex state_mutex;
513 struct completion stop_completion;
514 struct completion ib_completion;
515 struct completion up_completion;
516 struct list_head conn_list;
517 struct iser_login_desc login_desc;
518 unsigned int rx_desc_head;
519 struct iser_rx_desc *rx_descs;
520 u32 num_rx_descs;
521 unsigned short scsi_sg_tablesize;
522 unsigned int scsi_max_sectors;
523 bool snd_w_inv;
524};
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539struct iscsi_iser_task {
540 struct iser_tx_desc desc;
541 struct iser_conn *iser_conn;
542 enum iser_task_status status;
543 struct scsi_cmnd *sc;
544 int command_sent;
545 int dir[ISER_DIRS_NUM];
546 struct iser_mem_reg rdma_reg[ISER_DIRS_NUM];
547 struct iser_data_buf data[ISER_DIRS_NUM];
548 struct iser_data_buf prot[ISER_DIRS_NUM];
549};
550
551struct iser_page_vec {
552 u64 *pages;
553 int npages;
554 struct ib_mr fake_mr;
555};
556
557
558
559
560
561
562
563
564
565
566struct iser_global {
567 struct mutex device_list_mutex;
568 struct list_head device_list;
569 struct mutex connlist_mutex;
570 struct list_head connlist;
571 struct kmem_cache *desc_cache;
572};
573
574extern struct iser_global ig;
575extern int iser_debug_level;
576extern bool iser_pi_enable;
577extern int iser_pi_guard;
578extern unsigned int iser_max_sectors;
579extern bool iser_always_reg;
580
581int iser_assign_reg_ops(struct iser_device *device);
582
583int iser_send_control(struct iscsi_conn *conn,
584 struct iscsi_task *task);
585
586int iser_send_command(struct iscsi_conn *conn,
587 struct iscsi_task *task);
588
589int iser_send_data_out(struct iscsi_conn *conn,
590 struct iscsi_task *task,
591 struct iscsi_data *hdr);
592
593void iscsi_iser_recv(struct iscsi_conn *conn,
594 struct iscsi_hdr *hdr,
595 char *rx_data,
596 int rx_data_len);
597
598void iser_conn_init(struct iser_conn *iser_conn);
599
600void iser_conn_release(struct iser_conn *iser_conn);
601
602int iser_conn_terminate(struct iser_conn *iser_conn);
603
604void iser_release_work(struct work_struct *work);
605
606void iser_err_comp(struct ib_wc *wc, const char *type);
607void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc);
608void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc);
609void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc);
610void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc);
611void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc);
612void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc);
613
614void iser_task_rdma_init(struct iscsi_iser_task *task);
615
616void iser_task_rdma_finalize(struct iscsi_iser_task *task);
617
618void iser_free_rx_descriptors(struct iser_conn *iser_conn);
619
620void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
621 struct iser_data_buf *mem,
622 enum iser_data_dir cmd_dir);
623
624int iser_reg_rdma_mem(struct iscsi_iser_task *task,
625 enum iser_data_dir dir,
626 bool all_imm);
627void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
628 enum iser_data_dir dir);
629
630int iser_connect(struct iser_conn *iser_conn,
631 struct sockaddr *src_addr,
632 struct sockaddr *dst_addr,
633 int non_blocking);
634
635void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
636 enum iser_data_dir cmd_dir);
637void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
638 enum iser_data_dir cmd_dir);
639
640int iser_post_recvl(struct iser_conn *iser_conn);
641int iser_post_recvm(struct iser_conn *iser_conn, int count);
642int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
643 bool signal);
644
645int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
646 struct iser_data_buf *data,
647 enum iser_data_dir iser_dir,
648 enum dma_data_direction dma_dir);
649
650void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
651 struct iser_data_buf *data,
652 enum dma_data_direction dir);
653
654int iser_initialize_task_headers(struct iscsi_task *task,
655 struct iser_tx_desc *tx_desc);
656int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
657 struct iscsi_session *session);
658int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
659 unsigned cmds_max,
660 unsigned int size);
661void iser_free_fmr_pool(struct ib_conn *ib_conn);
662int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
663 unsigned cmds_max,
664 unsigned int size);
665void iser_free_fastreg_pool(struct ib_conn *ib_conn);
666u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
667 enum iser_data_dir cmd_dir, sector_t *sector);
668struct iser_fr_desc *
669iser_reg_desc_get_fr(struct ib_conn *ib_conn);
670void
671iser_reg_desc_put_fr(struct ib_conn *ib_conn,
672 struct iser_fr_desc *desc);
673struct iser_fr_desc *
674iser_reg_desc_get_fmr(struct ib_conn *ib_conn);
675void
676iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
677 struct iser_fr_desc *desc);
678
679static inline struct ib_send_wr *
680iser_tx_next_wr(struct iser_tx_desc *tx_desc)
681{
682 struct ib_send_wr *cur_wr = &tx_desc->wrs[tx_desc->wr_idx].send;
683 struct ib_send_wr *last_wr;
684
685 if (tx_desc->wr_idx) {
686 last_wr = &tx_desc->wrs[tx_desc->wr_idx - 1].send;
687 last_wr->next = cur_wr;
688 }
689 tx_desc->wr_idx++;
690
691 return cur_wr;
692}
693
694static inline struct iser_conn *
695to_iser_conn(struct ib_conn *ib_conn)
696{
697 return container_of(ib_conn, struct iser_conn, ib_conn);
698}
699
700static inline struct iser_rx_desc *
701iser_rx(struct ib_cqe *cqe)
702{
703 return container_of(cqe, struct iser_rx_desc, cqe);
704}
705
706static inline struct iser_tx_desc *
707iser_tx(struct ib_cqe *cqe)
708{
709 return container_of(cqe, struct iser_tx_desc, cqe);
710}
711
712static inline struct iser_login_desc *
713iser_login(struct ib_cqe *cqe)
714{
715 return container_of(cqe, struct iser_login_desc, cqe);
716}
717
718#endif
719