1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#ifndef __ISCSI_ISER_H__
42#define __ISCSI_ISER_H__
43
44#include <linux/types.h>
45#include <linux/net.h>
46#include <linux/printk.h>
47#include <scsi/libiscsi.h>
48#include <scsi/scsi_transport_iscsi.h>
49#include <scsi/scsi_cmnd.h>
50#include <scsi/scsi_device.h>
51
52#include <linux/interrupt.h>
53#include <linux/wait.h>
54#include <linux/sched.h>
55#include <linux/list.h>
56#include <linux/slab.h>
57#include <linux/dma-mapping.h>
58#include <linux/mutex.h>
59#include <linux/mempool.h>
60#include <linux/uio.h>
61
62#include <linux/socket.h>
63#include <linux/in.h>
64#include <linux/in6.h>
65
66#include <rdma/ib_verbs.h>
67#include <rdma/ib_fmr_pool.h>
68#include <rdma/rdma_cm.h>
69
70#define DRV_NAME "iser"
71#define PFX DRV_NAME ": "
72#define DRV_VER "1.5"
73
74#define iser_dbg(fmt, arg...) \
75 do { \
76 if (unlikely(iser_debug_level > 2)) \
77 printk(KERN_DEBUG PFX "%s: " fmt,\
78 __func__ , ## arg); \
79 } while (0)
80
81#define iser_warn(fmt, arg...) \
82 do { \
83 if (unlikely(iser_debug_level > 0)) \
84 pr_warn(PFX "%s: " fmt, \
85 __func__ , ## arg); \
86 } while (0)
87
88#define iser_info(fmt, arg...) \
89 do { \
90 if (unlikely(iser_debug_level > 1)) \
91 pr_info(PFX "%s: " fmt, \
92 __func__ , ## arg); \
93 } while (0)
94
95#define iser_err(fmt, arg...) \
96 pr_err(PFX "%s: " fmt, __func__ , ## arg)
97
98#define SHIFT_4K 12
99#define SIZE_4K (1ULL << SHIFT_4K)
100#define MASK_4K (~(SIZE_4K-1))
101
102#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
103#define ISER_DEF_XMIT_CMDS_DEFAULT 512
104#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
105 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
106#else
107 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
108#endif
109#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
110
111
112
113#define ISER_MAX_RX_MISC_PDUS 4
114
115#define ISER_MAX_TX_MISC_PDUS 6
116
117
118#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
119
120#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
121
122
123
124
125
126
127
128#define ISER_INFLIGHT_DATAOUTS 8
129
130#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
131 (1 + ISER_INFLIGHT_DATAOUTS) + \
132 ISER_MAX_TX_MISC_PDUS + \
133 ISER_MAX_RX_MISC_PDUS)
134
135
136#define ISER_MAX_REG_WR_PER_CMD 5
137
138
139#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
140 (1 + ISER_MAX_REG_WR_PER_CMD) + \
141 ISER_MAX_TX_MISC_PDUS + \
142 ISER_MAX_RX_MISC_PDUS)
143
144#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
145 - ISER_MAX_TX_MISC_PDUS \
146 - ISER_MAX_RX_MISC_PDUS) / \
147 (1 + ISER_INFLIGHT_DATAOUTS))
148
149#define ISER_WC_BATCH_COUNT 16
150#define ISER_SIGNAL_CMD_COUNT 32
151
152#define ISER_VER 0x10
153#define ISER_WSV 0x08
154#define ISER_RSV 0x04
155
156#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL
157#define ISER_BEACON_WRID 0xfffffffffffffffeULL
158
159
160
161
162
163
164
165
166
167
168
169struct iser_hdr {
170 u8 flags;
171 u8 rsvd[3];
172 __be32 write_stag;
173 __be64 write_va;
174 __be32 read_stag;
175 __be64 read_va;
176} __attribute__((packed));
177
178
179#define ISER_ZBVA_NOT_SUPPORTED 0x80
180#define ISER_SEND_W_INV_NOT_SUPPORTED 0x40
181
182struct iser_cm_hdr {
183 u8 flags;
184 u8 rsvd[3];
185} __packed;
186
187
188#define ISER_HEADERS_LEN (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
189
190#define ISER_RECV_DATA_SEG_LEN 128
191#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
192#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
193
194
195#define ISER_OBJECT_NAME_SIZE 64
196
197enum iser_conn_state {
198 ISER_CONN_INIT,
199 ISER_CONN_PENDING,
200 ISER_CONN_UP,
201 ISER_CONN_TERMINATING,
202 ISER_CONN_DOWN,
203 ISER_CONN_STATES_NUM
204};
205
206enum iser_task_status {
207 ISER_TASK_STATUS_INIT = 0,
208 ISER_TASK_STATUS_STARTED,
209 ISER_TASK_STATUS_COMPLETED
210};
211
212enum iser_data_dir {
213 ISER_DIR_IN = 0,
214 ISER_DIR_OUT,
215 ISER_DIRS_NUM
216};
217
218
219
220
221
222
223
224
225
226
227
228
229
230struct iser_data_buf {
231 void *buf;
232 unsigned int size;
233 unsigned long data_len;
234 unsigned int dma_nents;
235 char *copy_buf;
236 struct scatterlist sg_single;
237 };
238
239
240struct iser_device;
241struct iscsi_iser_task;
242struct iscsi_endpoint;
243
244
245
246
247
248
249
250
251
252
253struct iser_mem_reg {
254 u32 lkey;
255 u32 rkey;
256 u64 va;
257 u64 len;
258 void *mem_h;
259};
260
261
262
263
264
265
266
267
268
269
270struct iser_regd_buf {
271 struct iser_mem_reg reg;
272 void *virt_addr;
273 struct iser_device *device;
274 enum dma_data_direction direction;
275 unsigned int data_size;
276};
277
278enum iser_desc_type {
279 ISCSI_TX_CONTROL ,
280 ISCSI_TX_SCSI_COMMAND,
281 ISCSI_TX_DATAOUT
282};
283
284
285
286
287
288
289
290
291
292
293
294
295
296struct iser_tx_desc {
297 struct iser_hdr iser_header;
298 struct iscsi_hdr iscsi_header;
299 enum iser_desc_type type;
300 u64 dma_addr;
301 struct ib_sge tx_sg[2];
302 int num_sge;
303};
304
305#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
306 sizeof(u64) + sizeof(struct ib_sge)))
307
308
309
310
311
312
313
314
315
316
317struct iser_rx_desc {
318 struct iser_hdr iser_header;
319 struct iscsi_hdr iscsi_header;
320 char data[ISER_RECV_DATA_SEG_LEN];
321 u64 dma_addr;
322 struct ib_sge rx_sg;
323 char pad[ISER_RX_PAD_SIZE];
324} __attribute__((packed));
325
326struct iser_conn;
327struct ib_conn;
328struct iscsi_iser_task;
329
330
331
332
333
334
335
336
337
338
339
340struct iser_comp {
341 struct iser_device *device;
342 struct ib_cq *cq;
343 struct ib_wc wcs[ISER_WC_BATCH_COUNT];
344 struct tasklet_struct tasklet;
345 int active_qps;
346};
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367struct iser_device {
368 struct ib_device *ib_device;
369 struct ib_pd *pd;
370 struct ib_device_attr dev_attr;
371 struct ib_mr *mr;
372 struct ib_event_handler event_handler;
373 struct list_head ig_list;
374 int refcount;
375 int comps_used;
376 struct iser_comp *comps;
377 int (*iser_alloc_rdma_reg_res)(struct ib_conn *ib_conn,
378 unsigned cmds_max);
379 void (*iser_free_rdma_reg_res)(struct ib_conn *ib_conn);
380 int (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task,
381 enum iser_data_dir cmd_dir);
382 void (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
383 enum iser_data_dir cmd_dir);
384};
385
386#define ISER_CHECK_GUARD 0xc0
387#define ISER_CHECK_REFTAG 0x0f
388#define ISER_CHECK_APPTAG 0x30
389
390enum iser_reg_indicator {
391 ISER_DATA_KEY_VALID = 1 << 0,
392 ISER_PROT_KEY_VALID = 1 << 1,
393 ISER_SIG_KEY_VALID = 1 << 2,
394 ISER_FASTREG_PROTECTED = 1 << 3,
395};
396
397
398
399
400
401
402
403
404struct iser_pi_context {
405 struct ib_mr *prot_mr;
406 struct ib_fast_reg_page_list *prot_frpl;
407 struct ib_mr *sig_mr;
408};
409
410
411
412
413
414
415
416
417
418
419struct fast_reg_descriptor {
420 struct list_head list;
421 struct ib_mr *data_mr;
422 struct ib_fast_reg_page_list *data_frpl;
423 struct iser_pi_context *pi_ctx;
424 u8 reg_indicators;
425};
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450struct ib_conn {
451 struct rdma_cm_id *cma_id;
452 struct ib_qp *qp;
453 int post_recv_buf_count;
454 u8 sig_count;
455 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
456 struct iser_device *device;
457 struct iser_comp *comp;
458 bool pi_support;
459 struct ib_send_wr beacon;
460 struct completion flush_comp;
461 spinlock_t lock;
462 union {
463 struct {
464 struct ib_fmr_pool *pool;
465 struct iser_page_vec *page_vec;
466 } fmr;
467 struct {
468 struct list_head pool;
469 int pool_size;
470 } fastreg;
471 };
472};
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503struct iser_conn {
504 struct ib_conn ib_conn;
505 struct iscsi_conn *iscsi_conn;
506 struct iscsi_endpoint *ep;
507 enum iser_conn_state state;
508 unsigned qp_max_recv_dtos;
509 unsigned qp_max_recv_dtos_mask;
510 unsigned min_posted_rx;
511 u16 max_cmds;
512 char name[ISER_OBJECT_NAME_SIZE];
513 struct work_struct release_work;
514 struct mutex state_mutex;
515 struct completion stop_completion;
516 struct completion ib_completion;
517 struct completion up_completion;
518 struct list_head conn_list;
519
520 char *login_buf;
521 char *login_req_buf, *login_resp_buf;
522 u64 login_req_dma, login_resp_dma;
523 unsigned int rx_desc_head;
524 struct iser_rx_desc *rx_descs;
525 u32 num_rx_descs;
526};
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543struct iscsi_iser_task {
544 struct iser_tx_desc desc;
545 struct iser_conn *iser_conn;
546 enum iser_task_status status;
547 struct scsi_cmnd *sc;
548 int command_sent;
549 int dir[ISER_DIRS_NUM];
550 struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];
551 struct iser_data_buf data[ISER_DIRS_NUM];
552 struct iser_data_buf data_copy[ISER_DIRS_NUM];
553 struct iser_data_buf prot[ISER_DIRS_NUM];
554 struct iser_data_buf prot_copy[ISER_DIRS_NUM];
555};
556
557struct iser_page_vec {
558 u64 *pages;
559 int length;
560 int offset;
561 int data_size;
562};
563
564
565
566
567
568
569
570
571
572
573struct iser_global {
574 struct mutex device_list_mutex;
575 struct list_head device_list;
576 struct mutex connlist_mutex;
577 struct list_head connlist;
578 struct kmem_cache *desc_cache;
579};
580
581extern struct iser_global ig;
582extern int iser_debug_level;
583extern bool iser_pi_enable;
584extern int iser_pi_guard;
585
586int iser_send_control(struct iscsi_conn *conn,
587 struct iscsi_task *task);
588
589int iser_send_command(struct iscsi_conn *conn,
590 struct iscsi_task *task);
591
592int iser_send_data_out(struct iscsi_conn *conn,
593 struct iscsi_task *task,
594 struct iscsi_data *hdr);
595
596void iscsi_iser_recv(struct iscsi_conn *conn,
597 struct iscsi_hdr *hdr,
598 char *rx_data,
599 int rx_data_len);
600
601void iser_conn_init(struct iser_conn *iser_conn);
602
603void iser_conn_release(struct iser_conn *iser_conn);
604
605int iser_conn_terminate(struct iser_conn *iser_conn);
606
607void iser_release_work(struct work_struct *work);
608
609void iser_rcv_completion(struct iser_rx_desc *desc,
610 unsigned long dto_xfer_len,
611 struct ib_conn *ib_conn);
612
613void iser_snd_completion(struct iser_tx_desc *desc,
614 struct ib_conn *ib_conn);
615
616void iser_task_rdma_init(struct iscsi_iser_task *task);
617
618void iser_task_rdma_finalize(struct iscsi_iser_task *task);
619
620void iser_free_rx_descriptors(struct iser_conn *iser_conn);
621
622void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
623 struct iser_data_buf *mem,
624 struct iser_data_buf *mem_copy,
625 enum iser_data_dir cmd_dir);
626
627int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
628 enum iser_data_dir cmd_dir);
629int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
630 enum iser_data_dir cmd_dir);
631
632int iser_connect(struct iser_conn *iser_conn,
633 struct sockaddr *src_addr,
634 struct sockaddr *dst_addr,
635 int non_blocking);
636
637int iser_reg_page_vec(struct ib_conn *ib_conn,
638 struct iser_page_vec *page_vec,
639 struct iser_mem_reg *mem_reg);
640
641void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
642 enum iser_data_dir cmd_dir);
643void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
644 enum iser_data_dir cmd_dir);
645
646int iser_post_recvl(struct iser_conn *iser_conn);
647int iser_post_recvm(struct iser_conn *iser_conn, int count);
648int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
649 bool signal);
650
651int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
652 struct iser_data_buf *data,
653 enum iser_data_dir iser_dir,
654 enum dma_data_direction dma_dir);
655
656void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
657 struct iser_data_buf *data);
658int iser_initialize_task_headers(struct iscsi_task *task,
659 struct iser_tx_desc *tx_desc);
660int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
661 struct iscsi_session *session);
662int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max);
663void iser_free_fmr_pool(struct ib_conn *ib_conn);
664int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max);
665void iser_free_fastreg_pool(struct ib_conn *ib_conn);
666u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
667 enum iser_data_dir cmd_dir, sector_t *sector);
668#endif
669